diff --git a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
index aa7f125..1274afd 100644
--- a/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
+++ b/annotations/us_full_run_20260612_policyengine_4_16_1_populace/us_audit_row_annotations.csv
@@ -1,34 +1,36 @@
 country,scenario_id,variable,model,failure_source,failure_subtype,reference_suspect,annotation
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"It treated the filer as a qualifying surviving spouse and used joint Social Security thresholds, a joint standard deduction, and joint brackets despite no qualifying child being listed. The single-filer Social Security calculation includes $18,542.70, and the applicable itemized and senior deductions leave $26,095.78 taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It incorrectly included $48,000 of tax-exempt veterans benefits, deducted ESI premiums without wage income, and then asserted that deductions eliminated tax despite its own calculations showing substantial taxable income. The correct AGI is $50,437.11 and deductions leave $26,095.78 taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It taxed 85% of Social Security rather than applying the statutory single-filer worksheet, which yields $18,542.70, and it chose the standard deduction instead of $18,341.33 of itemized deductions. Those errors raised its taxable income above the correct $26,095.78."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"It assigned qualifying-surviving-spouse status without a listed qualifying child and then concluded that joint deductions reduced taxable income to zero. The filer is taxed under the single pathway, with $26,095.78 remaining after itemized and senior deductions."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its reasoning correctly identified single status but taxed 85% of Social Security, rejected the applicable itemized deduction, and then submitted $1,176 despite deriving tax near $3,300. The correct taxable Social Security and deductions yield $26,095.78 of taxable income and $2,883.49 of tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"It assumed qualifying-surviving-spouse status without a qualifying child, used joint Social Security thresholds and a joint standard deduction, and omitted the $6,000 senior deduction from its final arithmetic. The applicable single-filer computation instead produces $2,883.49."
-us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It first computed about $2,023 from its stated taxable income at the 10% joint rate, then replaced that result with $3,204 using a nonexistent higher-bracket adjustment. Its submitted amount does not follow its own bracket calculation, while the applicable brackets on $26,095.78 yield $2,883.49."
-us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It taxed the full 85% cap of Social Security and used only an aged standard deduction, omitting both the applicable $18,341.33 itemized deduction and the $6,000 senior deduction. The correct deductions reduce taxable income to $26,095.78."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It applied a TCJA-expiration regime with a personal exemption and treated ESI premiums as deductible medical expenses. The applicable 2026 computation instead uses $18,341.33 of itemized deductions plus the $6,000 senior deduction, with no personal exemption."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"It stated taxable income of $37,990 but submitted only $1,109 of tax, an amount incompatible with the applicable progressive brackets. It also failed to apply the deductions that reduce taxable income to $26,095.78, on which tax is $2,883.49."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It subtracted a nonexistent 2026 personal exemption and omitted the $6,000 senior deduction while using an incomplete itemized-deduction amount. The correct deduction total is $24,341.33, leaving $26,095.78 taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It assumed pre-TCJA rules, subtracted a personal exemption, and counted only $16,470 of itemized deductions. The applicable calculation has no personal exemption and uses $18,341.33 of itemized deductions plus the $6,000 senior deduction."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It subtracted a personal exemption and omitted the $6,000 senior deduction, producing excessive taxable income. The correct total deductions are $24,341.33 and taxable income is $26,095.78."
-us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It used a personal exemption and an incomplete itemized-deduction calculation while omitting the $6,000 senior deduction. Applying the correct $24,341.33 deduction total produces $26,095.78 of taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It correctly used single Social Security thresholds but chose an aged standard deduction after imposing an obsolete $10,000 SALT cap. The applicable itemized deductions are $18,341.33, and adding the $6,000 senior deduction reduces taxable income to $26,095.78."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It asserted that age-related deductions and a surviving-spouse rate structure eliminated taxable income without calculating either. The correct deductions leave $26,095.78 taxable income, so the liability is not zero."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It treated the aged standard deduction as large enough to offset pension, IRA, interest, and taxable Social Security income. After the applicable itemized and senior deductions, $26,095.78 remains taxable."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used qualifying-surviving-spouse Social Security thresholds, joint deductions, and joint brackets despite the absence of a qualifying child. Single-filer treatment produces $18,542.70 of taxable Social Security and $2,883.49 of tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used an aged standard deduction plus the senior deduction rather than the larger applicable itemized deduction plus senior deduction. It also misstated taxable Social Security and AGI; the trace yields $50,437.11 of AGI and $26,095.78 of taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It chose a $24,150 standard-and-senior deduction package instead of $18,341.33 of itemized deductions plus the $6,000 senior deduction. That $191.33 deduction shortfall inflated taxable income and tax above $2,883.49."
-us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"Its $359 result reflects joint surviving-spouse treatment and the resulting oversized standard deduction. Without a listed qualifying child, the single-filer pathway leaves $26,095.78 taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted that senior deductions and credits exceeded taxable income without calculating the taxable portion of Social Security or the deduction limit. The correct computation leaves $26,095.78 taxable and has no nonrefundable credit reducing the $2,883.49 liability."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied a post-TCJA personal exemption and omitted the $6,000 senior deduction while using the wrong deduction structure. The applicable deductions are $18,341.33 itemized plus $6,000 senior, leaving $26,095.78 taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It improperly treated employer-sponsored premiums as deductible medical spending and omitted the $6,000 senior deduction. The correct itemized deductions are $18,341.33, and total deductions of $24,341.33 produce $2,883.49 of tax."
-us,scenario_000,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It chose the aged standard deduction instead of the applicable $18,341.33 itemized deduction. Including that itemized amount with the $6,000 senior deduction lowers taxable income by another $191.33 to $26,095.78."
-us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no value or explanation for the requested output, so the required numeric result was missing."
-us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It rejected itemizing by comparing $16,470 of property taxes with the standard deduction, omitting other trace-derived itemized amounts that bring itemized deductions to $18,341.33. Itemizing plus the $6,000 senior deduction yields $26,095.78 taxable income."
-us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"It reduced an already inconsistent tax estimate with an inapplicable lifetime-learning credit and a $750 elderly credit despite acknowledging the education credit did not apply. No nonrefundable credit offsets the tax, and the correct taxable-income computation yields $2,883.49."
-us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It excluded all Social Security by ignoring that tax-exempt interest and half the benefits enter provisional income, which is $50,520. That calculation makes $18,542.70 of Social Security taxable, and the filer is not entitled to joint qualifying-surviving-spouse treatment without a qualifying child."
-us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted that the aged single standard deduction eliminated taxable income without computing taxable Social Security or comparing itemized deductions. The correct calculation leaves $26,095.78 taxable and produces $2,883.49 of tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used qualifying-surviving-spouse Social Security thresholds, standard deduction, and brackets despite no dependent child being listed. Single treatment produces $18,542.70 of taxable Social Security, and the filer itemizes $18,341.33 before taking the separate $6,000 senior deduction."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly included tax-exempt veterans benefits in gross income, treated ESI premiums as an above-the-line deduction, and then discarded its own positive tax calculations to submit zero. The actual taxable-income computation leaves $26,095.78 after itemized and senior deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly selected single status but treated 85% of Social Security as automatically taxable and used the standard deduction. The Social Security worksheet yields $18,542.70, and $18,341.33 of itemized deductions plus the $6,000 senior deduction leave $26,095.78 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly granted qualifying-surviving-spouse status without a listed dependent child and then asserted that deductions erased tax on roughly $14,000 of income. Single-filer treatment and the applicable deductions leave $26,095.78 taxable income, not zero."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's reasoning derives tax of roughly $3,300 but submits $1,176 without any intervening rule or arithmetic supporting that reduction. Applying the traced taxable income of $26,095.78 to the 2026 brackets yields $2,883.49."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model assumed qualifying-surviving-spouse status and used MFJ deductions, Social Security thresholds, and brackets without a dependent child. The filer is treated as single, itemizes $18,341.33, and also receives the $6,000 senior deduction."
+us,scenario_000,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model incorrectly used qualifying-surviving-spouse parameters and then changed a correctly stated 10% calculation of about $2,023 into $3,204 without arithmetic support. The single-filer trace leaves $26,095.78 taxable and produces $2,883.49."
+us,scenario_000,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated the full 85% cap as taxable Social Security and omitted both the $18,341.33 itemized deduction election and the separate $6,000 senior deduction. Those rules reduce taxable income to $26,095.78 rather than $36,726."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a 2026 TCJA sunset, restored a personal exemption, and applied pre-TCJA 10%/15% brackets. The operative 2026 rules instead provide the $6,000 senior deduction and current-law brackets on $26,095.78."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The submitted $1,109 does not follow from the model's stated $37,990 taxable income under any bracket computation it identifies. The traced deductions leave $26,095.78 taxable income, whose 2026 bracket tax is $2,883.49."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and pre-TCJA 10%/15% brackets for 2026. It also omitted the separate $6,000 senior deduction from the traced itemized-deduction calculation."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied reverted pre-TCJA brackets and a personal exemption. The 2026 computation instead combines $18,341.33 of itemized deductions with the $6,000 senior deduction and taxes $26,095.78 under current brackets."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model substituted a restored personal exemption for the applicable $6,000 senior deduction and did not use the full $18,341.33 itemized amount. It consequently taxed the wrong base under the wrong 2026 framework."
+us,scenario_000,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly used a personal exemption and pre-TCJA 10%/15% rates. The applicable computation uses $18,341.33 of itemized deductions plus the $6,000 senior deduction, followed by current 2026 brackets."
+us,scenario_000,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly computed single-filer taxable Social Security but chose a projected standard deduction after comparing it only with SALT. The full itemized amount is $18,341.33, which is combined with the separate $6,000 senior deduction to produce $26,095.78 of taxable income."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age and surviving-spouse deductions fully offset income without computing either taxable Social Security or itemized deductions. The actual calculation retains $26,095.78 of taxable income and therefore cannot yield zero tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the age-related deductions as large enough to eliminate all taxable income. After $18,341.33 of itemized deductions and the $6,000 senior deduction, $26,095.78 remains taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used joint surviving-spouse Social Security thresholds and the joint standard deduction despite the absence of a dependent child. Single thresholds yield $18,542.70 of taxable Social Security, followed by itemization and the senior deduction."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model approximated taxable Social Security and used an age-based standard deduction instead of the traced $18,341.33 itemized deduction. Exact inputs leave $26,095.78 taxable and produce $2,883.49, not $2,912.34."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction plus the senior deduction instead of itemizing $18,341.33 plus taking the senior deduction. This overstated taxable income by about $190 and therefore overstated the tax."
+us,scenario_000,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's $359 answer reflects joint surviving-spouse deductions and rates rather than single treatment. With no dependent child listed, the calculation uses single Social Security thresholds and leaves $26,095.78 taxable after itemized and senior deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that senior deductions or credits exceed taxable income. No credit eliminates the liability, and $26,095.78 remains taxable after the applicable itemized and senior deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed post-TCJA personal exemptions and 10%/15% brackets, while also using an age add-on inconsistent with its qualifying-surviving-spouse label. The operative calculation uses current 2026 brackets and a $6,000 senior deduction on top of $18,341.33 itemized deductions."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and pre-TCJA 10%/15% brackets. The applicable 2026 deduction is the $6,000 senior deduction alongside $18,341.33 of itemized deductions, yielding a lower taxable base taxed under current brackets."
+us,scenario_000,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model treated all $8,389 of ESI premiums as deductible medical spending before the 7.5% floor and then failed to subtract a personal exemption it invoked. The trace recognizes total itemized deductions of $18,341.33 and a separate $6,000 senior deduction, leaving $26,095.78 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model chose the aged standard deduction plus the $6,000 senior deduction and overlooked that $18,341.33 of itemized deductions is larger than its standard-deduction estimate. Itemizing lowers taxable income to $26,095.78 and tax to $2,883.49."
+us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_000,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model compared the standard deduction only with $16,470 of real-estate tax and missed the additional deductible amount that brings itemized deductions to $18,341.33. Itemizing plus the $6,000 senior deduction leaves $26,095.78 taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented a lifetime-learning reduction and applied an elderly credit despite acknowledging the education credit was inapplicable and failing the elderly-credit income limits. It also falsely concluded that deductions eliminated tax even though $26,095.78 remains taxable."
+us,scenario_000,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model omitted tax-exempt interest from provisional income, using $37,472 instead of $50,520, and therefore included only $2,951.20 of Social Security. The single-filer Social Security worksheet yields $18,542.70 of taxable benefits."
+us,scenario_000,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared Social Security wholly nontaxable while omitting tax-exempt interest and half of Social Security from provisional income. Provisional income is $50,520, producing $18,542.70 of taxable Social Security, and qualifying-surviving-spouse status is unavailable without a dependent child."
+us,scenario_000,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that the aged single standard deduction erased taxable income without calculating the taxable portion of Social Security. Including $18,542.70 of taxable Social Security and applying the traced deductions leaves $26,095.78 taxable."
 us,scenario_000,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_000,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated age 77 and an asserted income exclusion calculation as sufficient for Texas aged Medicaid without identifying or satisfying an aged eligibility pathway. The person qualifies through no Medicaid category and receives no SSI, so the claimed below-threshold aged eligibility does not apply."
 us,scenario_000,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no earned wages were reported and that payroll tax therefore equals zero, but then submitted $1,534 instead. Its numeric output contradicts its own payroll-tax-base analysis and the required final-value contract."
@@ -66,36 +68,38 @@ us,scenario_002,federal_income_tax_before_refundable_credits,inkling,llm_error,t
 us,scenario_002,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Although the model ultimately excluded disability, it treated all $44,080 of Social Security as taxable merely because provisional income exceeded the first MFJ threshold. Provisional income of $40,780 remains below the $44,000 second threshold, making only $4,390 taxable; AGI is $23,130 and the joint standard deduction eliminates it."
 us,scenario_002,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified a zero taxable wage base and then contradicted that computation by inventing taxable wage-equivalent income and assigning $1,926 of Social Security and Medicare taxes. No listed income is employee compensation subject to payroll tax, so the employee Social Security, Medicare, Additional Medicare, and mandatory state payroll-tax components all equal zero."
 us,scenario_002,spouse_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly evaluated the spouse as a separate zero-income individual and assumed that age alone opened an aged Medicaid pathway. Washington's pathway evaluation includes the applicable household income and categorical requirements; this spouse receives no SSI, has household MAGI of 2.90 times FPL, and qualifies for no category."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning computed approximately $22,379 but submitted $27,049, so its output does not follow its own derivation. Its derivation also omitted the engine's $2,699.98 of above-the-line deductions and used the wrong standard-deduction and bracket parameters."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed only $23,384 of Social Security instead of $50,583.25 under the two-tier Social Security formula, and it improperly deducted a traditional 401(k) contribution despite zero wages. It also invented an age-65 additional standard deduction even though both spouses are under 65 and then asserted unidentified nonrefundable credits."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's stated calculations produced roughly $22,652–$22,674 but it submitted $19,524, an unsupported departure from its own arithmetic. It also failed to apply the trace's $2,699.98 of above-the-line deductions and exact 2026 bracket thresholds."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model denied all IRA-related adjustment because no wages were listed and therefore used AGI of $183,291 instead of $182,172.27. It also used estimated rather than exact 2026 bracket thresholds, overstating the tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the wrong 2026 married-filing-jointly bracket calculation, claiming about $23,886 of ordinary tax where the correct ordinary-bracket result is $21,590.70. It also omitted the $2,699.98 of above-the-line deductions."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a guessed $30,000 standard deduction instead of $32,200 and rejected the trace's above-the-line deductions, leaving taxable income too high. Its estimated bracket thresholds then compounded the overstatement."
-us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied only a $1,118 IRA deduction rather than the full $2,699.98 of above-the-line deductions and mis-summed pre-Social-Security income by $100. It then rounded an approximate bracket computation to $22,000 instead of calculating the exact liability."
-us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly applied pre-TCJA personal exemptions, the pre-TCJA 10%/15%/25% rate structure, and a $15,000 standard deduction. The applicable computation uses the $32,200 standard deduction, no personal exemptions, and the 2026 10%/12%/22% brackets at this income."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, substituted a $16,000 standard deduction plus personal exemptions, and used 10%/15%/25% ordinary rates. The applicable 2026 calculation uses a $32,200 standard deduction, no personal exemptions, and 10%/12%/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model described approximately $183,865 as taxable income even after claiming to account for deductions, effectively failing to subtract the $32,200 standard deduction and the trace's $2,699.98 of above-the-line deductions. That inflated tax base produced the excessive liability."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration of the governing rate and deduction regime and used personal exemptions with a reduced standard deduction. The correct computation uses the $32,200 standard deduction, no personal exemptions, and the applicable 2026 brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a post-sunset $16,100 standard deduction, personal exemptions, and pre-TCJA ordinary rates. The applicable calculation instead uses the $32,200 married-joint standard deduction, no personal exemptions, and the 2026 10%/12%/22% brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $30,800 standard deduction instead of $32,200 and omitted the $2,699.98 of above-the-line deductions, producing $152,490.50 rather than $149,972.27 of taxable income. It therefore overstated ordinary-rate tax."
-us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's reference to both a standard deduction and personal exemptions shows that it used the wrong 2026 tax regime. Personal exemptions do not enter this calculation; the applicable deduction is $32,200 and the applicable ordinary brackets yield $21,590.70 before the $564 preferential-rate amount."
-us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model incorrectly capped the $1,582 net capital loss at a $3,000 deduction, omitted tax-exempt interest from provisional income, invented a $500 sick-spouse credit, and used the wrong deduction and rates. More decisively, it stated a final calculation of $28,867 but submitted $49,523.45, which is disconnected from its own arithmetic."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated deductions as eliminating tax despite $149,972.27 of taxable income after the allowed adjustments and standard deduction. Neither spouse qualifies for an age-65 deduction, and the listed medical costs do not generate an itemized deduction large enough to replace the $32,200 standard deduction."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model reduced the liability to $2,320 by overstating retirement-account or itemized deductions. The trace leaves $149,972.27 taxable after all permitted above-the-line deductions and the standard deduction, and the medical expenses remain below the 7.5%-of-AGI floor."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used about $151,091 of taxable income instead of $149,972.27 because it omitted the trace's $2,699.98 of above-the-line deductions and used an approximate gross-income path. Applying the exact deduction and 2026 brackets lowers the result to $22,154.70."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The submitted amount matches a calculation based on roughly $151,091 of taxable income, not the traced $149,972.27. The model failed to incorporate the full $2,699.98 of above-the-line deductions before subtracting the $32,200 standard deduction."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly used $151,091 of taxable income rather than $149,972.27. It omitted the full effect of the $2,699.98 above-the-line adjustment in the traced AGI calculation."
-us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model explicitly used $151,090.50 of taxable income instead of $149,972.27. Its income calculation did not apply the trace's full $2,699.98 of above-the-line deductions before the standard deduction."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated retirement contributions and other deductions as sufficient to erase tax, but the permitted adjustments and $32,200 standard deduction leave $149,972.27 taxable. No listed nonrefundable credit offsets the resulting $22,154.70 liability."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset regime with personal exemptions, a $16,100 standard deduction, and 10%/15%/25% rates. The applicable computation uses no personal exemptions, a $32,200 standard deduction, and the 2026 10%/12%/22% ordinary brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used inflation-adjusted pre-TCJA personal exemptions, a reduced standard deduction, and pre-TCJA brackets. Those provisions do not govern this calculation, which uses the $32,200 standard deduction, no personal exemptions, and the applicable 2026 brackets."
-us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as about $184,371, whereas the traced AGI is $182,172.27 after $2,699.98 of above-the-line deductions. This carried through to taxable income of about $152,171 instead of $149,972.27."
-us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly projected a pre-TCJA regime with a $16,950 standard deduction, personal exemptions, and 10%/15%/25% brackets. The applicable 2026 rules use the $32,200 standard deduction, no personal exemptions, and 10%/12%/22% rates at this income."
-us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $500 deduction for auto-loan interest and rejected the trace's other above-the-line adjustments, producing $150,590.50 instead of $149,972.27 of taxable income. The listed auto-loan interest is not the adjustment used in the traced federal calculation."
-us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the absence of wages as preventing a tax calculation and returned zero. The listed IRA distributions, pension, taxable Social Security, interest, dividends, and capital transactions independently produce $149,972.27 of taxable income and a positive liability."
-us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model failed to subtract the $1,582 net capital loss from gross income, treated the 64-year-old head as age 65+, and omitted the $2,699.98 above-the-line deduction. It then stated a calculated tax of $25,531.98 but submitted $41,228.50, which does not follow its own reasoning."
-us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's $21,902.90 result understates the tax generated by $149,972.27 of taxable income under the exact 2026 married-joint brackets plus the $564 preferential-rate amount. Its terse estimate did not apply the governing thresholds and rates exactly."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It denied the $1,118 traditional IRA deduction and therefore used AGI around $183,291 instead of $182,172.27. Its submitted $27,049 also contradicts its own final calculation of roughly $22,379, adding an unexplained $4,670."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It deducted the nonexistent $23,926 traditional 401(k) deferral from zero wages, taxed only $23,384 of Social Security instead of $50,583.25, and used an age-based deduction even though neither spouse is 65. Those errors drastically understated taxable income."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It rejected the $1,118 traditional IRA deduction, leaving AGI $1,118 too high, and used estimated rather than applicable 2026 brackets. Its submitted $19,524 then contradicts its own computed range of $22,652–$22,674 without identifying any credit or adjustment."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It rejected the $1,118 traditional IRA deduction and used estimated bracket thresholds, producing taxable income of $150,991 instead of $149,972.27. Both errors raised the ordinary-rate tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It used the wrong 2026 married-filing-jointly bracket computation, claiming about $23,886 of ordinary tax on roughly $147,331 when the applicable computation yields $21,590.70 on the correct ordinary component. It also omitted the $1,118 IRA deduction."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It rejected the $1,118 IRA deduction and used a guessed $30,000 standard deduction instead of $32,200. This inflated taxable income by about $3,318 before it applied estimated bracket thresholds."
+us,scenario_003,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"It applied the IRA deduction but introduced a $100 error in other income and then rounded an approximate bracket calculation to $22,000. The exact derivation gives taxable income of $149,972.27 and total tax of $22,154.70."
+us,scenario_003,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It incorrectly applied pre-TCJA personal exemptions, a $15,000 standard deduction, and 10%/15%/25% brackets. The applicable 2026 calculation uses the $32,200 joint standard deduction and current 10%/12%/22% bracket structure at this income."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It incorrectly assumed TCJA expiration, substituting a $16,000 standard deduction, personal exemptions, and pre-TCJA 10%/15%/25% rates. It also omitted the $1,118 IRA deduction from AGI."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It described approximately $183,865 as taxable income even after deductions, effectively failing to subtract the applicable $32,200 standard deduction and the full above-the-line deductions. That shortcut produced a substantially overstated tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It incorrectly assumed TCJA expiration and used personal exemptions plus the smaller pre-TCJA standard deduction and rate schedule. It also began from AGI that omitted the $1,118 IRA deduction."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It incorrectly used a $16,100 standard deduction, $10,100 of personal exemptions, and post-sunset ordinary brackets. The applicable deduction is the $32,200 joint standard deduction without personal exemptions, and AGI also includes the $1,118 IRA deduction."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $1,118 traditional IRA deduction and used a $30,800 standard deduction instead of $32,200. These mistakes inflated taxable income from $149,972.27 to $152,490.50."
+us,scenario_003,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,It applied personal exemptions and a post-TCJA-sunset tax framework rather than the applicable 2026 joint standard deduction and brackets. That framework overstated the liability.
+us,scenario_003,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"It mis-netted the capital transactions by taking both the $1,080 short-term gain and a separate $3,000 loss, used the wrong deduction and rates, and invented a $500 sick-spouse credit. Its submitted $49,523.45 also bears no relation to its own stated final calculation of $28,867."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treated deductions as sufficient to erase tax despite $149,972.27 of taxable income after the $32,200 standard deduction. Neither spouse qualifies for an age-65 deduction, and the listed medical expenses do not exceed the 7.5%-of-AGI floor."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $2,320 result requires deductions or credits far beyond those provided. The correct adjustments leave $149,972.27 taxable, and no listed nonrefundable credit reduces the resulting $22,154.70 liability."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It used taxable income of about $151,091, reflecting omission of the $1,118 traditional IRA deduction and a small standard-deduction discrepancy. The correct taxable income is $149,972.27 before applying the preferential-rate computation."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"Its result tracks taxable income around $151,090.50, which omits the $1,118 traditional IRA deduction. Applying the full $2,699.98 of above-the-line deductions produces taxable income of $149,972.27."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It used taxable income of about $151,091 rather than $149,972.27 because it omitted the $1,118 traditional IRA deduction. That overstatement carried into its ordinary-rate tax."
+us,scenario_003,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used taxable income of $151,090.50, exactly reflecting AGI before the $1,118 traditional IRA deduction. The full above-the-line deductions reduce taxable income to $149,972.27."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It asserted that contributions and deductions erase the liability, but zero wages prevent a $23,926 401(k) deferral and the allowable deductions still leave $149,972.27 taxable. No listed nonrefundable credits reduce the tax to zero."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It incorrectly applied a post-TCJA-sunset standard deduction, personal exemptions, and 10%/15%/25% rates. The applicable 2026 calculation uses a $32,200 joint standard deduction, no personal exemptions, and the current bracket structure; it also omitted the IRA deduction."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It incorrectly used post-TCJA-sunset personal exemptions, the smaller standard deduction, and pre-TCJA ordinary rates. It also rejected the $1,118 traditional IRA deduction, overstating AGI."
+us,scenario_003,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It incorrectly substituted inflation-adjusted pre-TCJA standard deductions, personal exemptions, and 10%/15%/25% brackets. It also omitted the $1,118 traditional IRA deduction from AGI."
+us,scenario_003,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It overstated AGI by about $2,199, using $184,371 instead of $182,172.27. This reflects failure to apply the traced above-the-line deductions correctly and leads to taxable income about $2,199 too high."
+us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It incorrectly applied a projected pre-TCJA standard deduction, personal exemptions, and 10%/15%/25% brackets. It also denied the $1,118 traditional IRA deduction, leaving AGI too high."
+us,scenario_003,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It denied the $1,118 traditional IRA deduction and instead invented a $500 car-loan-interest deduction. The traced adjustments total $2,699.98 and contain no such car-loan-interest deduction, yielding taxable income of $149,972.27."
+us,scenario_003,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated absent wages as making the tax base indeterminate and returned zero, ignoring the taxable IRA distributions, pension, Social Security, interest, and dividends. Those items produce $149,972.27 of taxable income after deductions."
+us,scenario_003,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It double-counted the $1,582 capital-loss effect: the component sum already included the $1,080 gain and $2,662 loss, then it subtracted another $1,582. It also omitted the $1,118 IRA deduction, causing its stated taxable income to differ from the traced $149,972.27."
+us,scenario_003,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"It failed to subtract the $2,699.98 of above-the-line deductions, treated the 64-year-old head as age 65+, and used the wrong ordinary-tax computation. Its submitted $41,228.50 also contradicts its own stated total of $25,531.98."
+us,scenario_003,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"Its unexplained estimate understates the liability after combining the retirement income, taxable Social Security, investments, and deductions. The exact derivation leaves $149,972.27 taxable and produces $21,590.70 of ordinary-rate tax plus $564 of preferential-rate tax."
 us,scenario_003,head_medicare_eligible,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,The model replaced current-year Medicare eligibility with an invented near-term eligibility rule. Being age 64 and approaching 65 does not satisfy the age-65 threshold during a year in which age is constant.
 us,scenario_003,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The reasoning correctly stated that the age-64 head is not Medicare eligible, but the model submitted value = 1, reversing its own eligibility conclusion. The required numeric encoding was 0 for “not eligible.”"
 us,scenario_003,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a projection from age 64 to future age-65 eligibility. The benchmark holds age constant throughout 2026, and no disability or qualifying disease pathway is present, so the head fails every Medicare eligibility pathway."
@@ -117,127 +121,135 @@ us,scenario_004,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_
 us,scenario_004,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"After correctly deriving near-zero New York taxable income from the listed amounts, the model invented approximately $90,000 of wages from the spouses' work hours and median-wage assumptions. The prompt explicitly sets unlisted wage amounts to zero, so those imputed wages cannot enter New York taxable income."
 us,scenario_004,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,582 implies a positive New York taxable-income base, but the model gave no computation supporting one. New York's Social Security exclusion leaves $1,200 of interest minus the $135 partnership loss, and the joint standard deduction reduces that $1,065 remainder to zero."
 us,scenario_004,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model began its state calculation with the full $51,475 Social Security benefit in federal AGI and then failed to remove it consistently when deriving its asserted $18,426 taxable income. It also incorrectly added back the $721 traditional IRA contribution; after New York's Social Security subtraction, the remaining $1,065 is below the joint standard deduction, producing zero taxable income."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model substituted an estimated $32,600 standard deduction and excluded the state refund based on an invented prior-year assumption instead of using the traced $5,163.84 above-the-line deduction and $33,269.77 taxable-income deduction. It also omitted the $5,721.98 NIIT, leaving its total below the required $106,505.90."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $110,000 of mortgage interest from the mortgage balance even though unlisted interest expense is zero, then produced a tax amount incompatible with its own approximately $408,000 taxable-income figure. It also treated employer insurance premiums and unspecified investment deductions as nonrefundable credits, although neither creates such a credit here."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly excluded NIIT from the requested output and then submitted $91,458 despite its own regular-tax calculation of about $102,907. The correct computation includes $89,095.92 of ordinary tax, $11,688 of preferential-rate tax, and $5,721.98 of NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from an assumed 7% rate and used an unsupported $60,000 itemized deduction. It then overstated both the qualified-dividend rate and the total tax; the traced deduction is $33,269.77 and the $77,920 preferential-rate amount generates $11,688."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model made an arithmetic error by calling the listed components approximately $604,318 of AGI; its own components net to $538,418 before the engine's specific deduction treatment. It also invented about $60,000 of itemized deductions and folded NIIT into an asserted 18.8% dividend rate rather than computing the three traced tax components separately."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA individual rules expired for 2026 and applied reconstructed pre-TCJA brackets, personal-era AMT rules, a restored miscellaneous deduction, and an invented 7% mortgage rate. The applicable ordinary brackets produce $89,095.92, no added AMT component appears in the trace, and NIIT is $5,721.98."
-us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented approximately $28,000 of mortgage interest from the balance and therefore used an unsupported $40,574 itemized deduction. The engine instead applies $33,269.77 of taxable-income deductions and reaches $503,494.75 of taxable income."
-us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model imposed a $131,247 tentative AMT in place of the traced ordinary and preferential-rate computation. The output contains $89,095.92 of ordinary-bracket tax, $11,688 of capital-gains-rate tax, and $5,721.98 of NIIT, with no separate AMT amount driving liability."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied an expired-rules framework with uncapped SALT, inferred mortgage interest, and personal exemptions, producing an unsupported $108,283 deduction plus $10,400 of exemptions. The traced taxable-income deduction is $33,269.77, which makes ordinary tax $89,095.92 rather than $88,016."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used $460,936 as AGI and described subtracting the standard deduction while computing AGI, conflating adjusted gross income with taxable income. The trace starts from $541,928.34 of gross income, subtracts $5,163.84 above the line and $33,269.77 afterward, and yields $503,494.75 of taxable income."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an incorrect roughly $525,049 AGI and added an AMT component not present in the traced liability. Applying the actual taxable-income computation and the ordinary, preferential, and NIIT components yields $106,505.90."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly treated the TCJA individual provisions as expired and applied uncapped SALT and pre-TCJA rates up to 33%. The applicable deduction and rate schedule produces $89,095.92 of ordinary tax, followed by $11,688 of preferential tax and $5,721.98 of NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model deducted $13,369 of employer-sponsored insurance premiums from wages in addition to the retirement deferrals, reducing income to an erroneous $525,049. It also invoked personal exemptions and taxed the qualified dividends at 20%, whereas the traced preferential tax is 15% of $77,920, or $11,688."
-us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted health-premium reductions against the reported wages and inferred mortgage interest despite the instruction that unlisted numeric inputs are zero. Those unsupported deductions drove the result far below the tax on $503,494.75 of traced taxable income."
-us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the 2025 standard deduction and 2025 ordinary brackets for a 2026 case. It also omitted the $5,721.98 NIIT; the correct ordinary and preferential taxes are $89,095.92 and $11,688 before NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $21,735 is incompatible with the model's own recognition that substantial ordinary income and qualified dividends remain taxable. Tax on the traced $503,494.75 of taxable income is $100,783.92 before the additional $5,721.98 NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied an unspecified bundle of itemized deductions and nonrefundable credits that the household facts do not support. No nonrefundable credit reduces the traced tax, and the applicable deductions leave $503,494.75 taxable."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model substituted a $32,200 standard deduction plus a $1,070 nonitemizer charitable deduction for the engine's $33,269.77 taxable-income deduction. That $1,653.77 taxable-income difference and the corresponding ordinary-bracket calculation caused its $529.02 overstatement."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The terse estimate did not apply the exact 2026 ordinary brackets to the traced $503,494.75 taxable income. The correct components are $89,095.92 ordinary tax, $11,688 preferential tax, and $5,721.98 NIIT."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions without identifying or matching the traced $33,269.77 deduction. Its resulting tax is $632.10 too high because the deduction and ordinary-bracket computation do not reproduce $89,095.92 of ordinary tax."
-us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used a generic married-joint standard deduction rather than the traced $33,269.77 taxable-income deduction. This left taxable income above $503,494.75 and overstated the ordinary-bracket component."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model introduced a qualified-business-income deduction even though the household has no listed qualified business income and referenced unspecified nonrefundable credits. The liability instead follows directly from $503,494.75 of taxable income with no nonrefundable credit reduction."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a post-sunset Pease, personal-exemption, miscellaneous-deduction, and AMT regime that does not govern this 2026 computation. Its $125,199 tentative AMT replaces the applicable $89,095.92 ordinary tax plus $11,688 preferential tax and therefore overstates liability."
-us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted $13,369 of employer health premiums from wages a second time, used an erroneous $525,049 AGI, and restored an unsupported miscellaneous itemized deduction. It also calculated $5,836 of NIIT from $153,578 rather than the traced $5,721.98 on $150,578.34 of net investment income."
-us,scenario_005,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model double-deducted employer health premiums, applied uncapped SALT and Pease, and added $18,221 of AMT. The traced computation uses $33,269.77 of deductions and contains no AMT increment; NIIT is $5,721.98."
-us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required value could not be parsed."
-us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $32,200 standard deduction and obtained $506,218 of taxable income instead of applying the traced $5,163.84 above-the-line deduction and $33,269.77 taxable-income deduction. The correct taxable income is $503,494.75, producing $89,095.92 rather than $89,967.36 of ordinary tax."
-us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented enough mortgage interest, SALT, charitable, employee-expense, and IRA deductions to erase all taxable income. The mortgage balance does not itself supply deductible interest, the tax-exempt pension does not offset taxable income, and the traced deductions leave $503,494.75 taxable."
-us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest using an assumed 5.75% rate and incorrectly reduced preferential income by the $10,911 net capital loss, leaving only $67,009 at capital-gains rates. The trace retains $77,920 of adjusted net capital gains, taxes it at 15% for $11,688, and separately applies the capital-loss treatment in arriving at taxable income."
-us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used a $15,700 standard deduction, the single-filer amount, despite identifying the household as married filing jointly. It also used erroneous gross and taxable income figures and failed to reproduce the separate $11,688 preferential tax and $5,721.98 NIIT components."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $46,308 of desired traditional 401(k) contributions and denied both traditional IRA deductions, instead of using the engine’s $41,650 employment-income reduction and $5,163.84 total above-the-line deductions. It also used an estimated $32,600 deduction rather than the traced $33,269.77 and omitted the $5,721.98 NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $110,000 of mortgage interest even though no mortgage-interest expense was listed, treated employer insurance premiums as tax credits or investment deductions, and never produced a coherent tax computation from its stated taxable income. The traced computation instead uses $33,269.77 of taxable-income deductions and taxes $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly excluded NIIT from the requested federal-income-tax output, omitting $5,721.98 that PolicyEngine includes. Its final $91,458 also contradicts its own stated regular-tax calculation of approximately $102,907."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from an assumed 7% rate and used an estimated $60,000 itemized deduction despite the instruction that unlisted expenses are zero. It also assigned qualified dividends a 20% rate rather than the traced $11,688 at 15% and did not compute the traced NIIT separately."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model made an arithmetic error by calling its listed income components approximately $604,318; those components do not sum to that AGI. It then invented about $60,000 of mortgage-related itemized deductions instead of using the traced $33,269.77 taxable-income deduction."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied a supposed 2026 TCJA sunset, restored pre-TCJA ordinary rates and miscellaneous deductions, and constructed an AMT liability from those obsolete assumptions. The applicable 2026 rules yield $89,095.92 of ordinary tax with no added AMT component in the traced result."
+us,scenario_005,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented approximately $28,000 of mortgage interest and therefore used a $40,574 itemized deduction. With no listed mortgage-interest expense, the traced taxable-income deduction is $33,269.77, and taxable income is $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model replaced the regular-tax result with a $131,247 tentative AMT even though the traced calculation contains $89,095.92 of ordinary tax, $11,688 of preferential-rate tax, and no AMT increment. Adding NIIT to that erroneous AMT produced its overstatement."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an assumed TCJA expiration, uncapped SALT deductions, a $1 million mortgage-interest limit, and personal exemptions. Those provisions do not govern the traced 2026 calculation, which uses $33,269.77 of deductions and $503,494.75 of taxable income."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $460,936 AGI that does not follow from the household’s taxable wages and investment income and described the tax-exempt pension as part of that calculation. The traced path begins with $541,928.34 of gross income and reaches $503,494.75 of taxable income after specified deductions."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used roughly $525,049 of AGI and added AMT without deriving either figure from the listed inputs. The traced computation uses $541,928.34 of gross income, $5,163.84 of above-the-line deductions, and no AMT increment."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly treated the 2026 tax year as governed by expired-TCJA rules, including uncapped SALT and ordinary rates reaching 33% at this income. The applicable brackets generate $89,095.92 of ordinary tax on the traced taxable-income base."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted employer-sponsored insurance premiums from wages even though the engine’s employment-income calculation reduces wages only to $388,350, and it added personal exemptions. It also taxed qualified dividends at 20% instead of the traced 15%, or $11,688."
+us,scenario_005,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model deducted both unlisted mortgage interest and employer health premiums from the reported wages, materially understating the tax base. The traced taxable income is $503,494.75 before applying preferential dividend tax and NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,period_annualization,False,"The model applied 2025 brackets and the 2025 $30,000 standard deduction to a 2026 case. It also omitted the $5,721.98 NIIT and used the full desired 401(k) deferrals instead of the engine’s $41,650 employment-income reduction."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The submitted $21,735 is incompatible with the model’s own acknowledgment that substantial ordinary-income tax, preferential-rate tax, and NIIT remain. The traced components alone are $89,095.92, $11,688, and $5,721.98."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The answer implies excessive deductions or credits because the household has $503,494.75 of traced taxable income and no applicable nonrefundable credits. Real-estate taxes and medical expenses do not reduce the liability to $75,657 under the applicable deduction rules."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $32,200 standard deduction plus a separate $1,070 nonitemizer charitable deduction, producing $505,148 rather than the traced $503,494.75 of taxable income. The applicable taxable-income deduction is $33,269.77, and the ordinary tax is $89,095.92."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The answer implies that the 2026 ordinary brackets were approximated too low after using the standard deduction. On the traced $503,494.75 taxable income, ordinary tax is $89,095.92 before adding $11,688 of preferential-rate tax and $5,721.98 of NIIT."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions without identifying the allowed amount and arrived above the traced liability. The correct deduction stage uses $5,163.84 above the line and $33,269.77 against taxable income, yielding $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the full desired traditional 401(k) deferrals and a generic standard deduction, yielding the same overstated result as calculations based on $538,418 of AGI. The traced employment-income reduction is $41,650, total above-the-line deductions are $5,163.84, and taxable income is $503,494.75."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model applied a qualified-business-income deduction even though the household has no listed business or self-employment income. It also invoked unspecified mortgage interest and credits rather than deriving the $33,269.77 deduction and zero nonrefundable credits used in the trace."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied post-sunset Pease, personal-exemption, miscellaneous-deduction, and AMT rules that do not govern this 2026 computation. Its $125,199 tentative AMT replaces the traced $89,095.92 ordinary tax without a valid AMT increment."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model imposed sunset-era itemized-deduction and Pease rules and added $25,486 of AMT. The traced calculation instead uses $33,269.77 of taxable-income deductions and contains no AMT addition."
+us,scenario_005,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $13,369 of employer health premiums from the reported wages and restored an unreimbursed-employee-expense deduction. Those steps produced the wrong AGI and deduction base; the trace uses $388,350 of employment income and $503,494.75 of taxable income."
+us,scenario_005,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied uncapped SALT, Pease, restored miscellaneous deductions, and an $18,221 AMT increment. None of those adjustments appears in the applicable traced computation, whose ordinary-tax component is $89,095.92."
+us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for the requested output.
+us,scenario_005,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted the full $46,308 of desired 401(k) contributions and denied the traditional IRA deductions, leading to $538,418 of AGI and $506,218 of taxable income. The trace instead uses a $41,650 employment-income reduction, $5,163.84 of above-the-line deductions, and $503,494.75 of taxable income."
+us,scenario_005,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented enough mortgage interest and other itemized deductions to erase more than half a million dollars of income. The listed mortgage balance is not mortgage-interest expense, and the traced deductions leave $503,494.75 taxable rather than zero."
+us,scenario_005,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model deducted the full $46,308 of desired 401(k) contributions and used a $32,200 deduction, producing $506,218 of taxable income instead of $503,494.75. It also understated NIIT by using $142,667 rather than the traced $150,578.34 of net investment income."
+us,scenario_005,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest using an assumed 5.75% rate and treated only $67,009 as preferential-rate income by netting the capital loss against qualified dividends. The trace has $77,920 of adjusted net capital gains taxed at 15%, $33,269.77 of deductions, and an additional $5,721.98 NIIT that the model omitted."
+us,scenario_005,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used the $15,700 single standard deduction for a married-filing-jointly household and started from an unsupported $528,543 gross-income figure. The traced joint-filer calculation uses $33,269.77 of taxable-income deductions and produces $503,494.75 of taxable income."
 us,scenario_005,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_005,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model used a $183,600 Social Security wage base instead of $184,500 and a 1.2% California SDI rate instead of 1.3%. Its submitted $33,296.55 also contradicts its own revised calculation of $35,558.20."
-us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used an obsolete $168,600 Social Security cap, incorrectly capped California SDI, added employee ETT, and then improperly included $10,086.52 of state income tax in payroll tax. Payroll tax excludes state individual income tax and includes uncapped 1.3% California SDI."
-us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model used a $181,800 Social Security wage base and a 1.2% California SDI rate. The applicable inputs are a $184,500 Social Security wage base and uncapped SDI of 1.3%, yielding $22,599 of Social Security and $5,590 of SDI."
-us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model correctly derived $22,599 of Social Security, $6,235 of Medicare, and $1,620 of Additional Medicare Tax but submitted less than even that federal subtotal and omitted California SDI. California SDI is $5,590 at 1.3% of $430,000."
-us,scenario_005,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model named the correct federal components but failed to add California SDI coherently, claiming it was already reflected without showing where. The federal subtotal is $30,454 and uncapped California SDI adds $5,590."
-us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model used the obsolete $176,100 Social Security wage base for both spouses and a 1.2% California SDI rate. The 2026 wage base is $184,500, leaving the head fully taxable for Social Security, and SDI is 1.3% of gross wages."
-us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The submitted $27,306 does not follow any of the model’s displayed totals and excludes required payroll-tax components. It also used a $176,100 Social Security cap and 1.1% SDI rather than the $184,500 cap and 1.3% uncapped SDI rate."
-us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model understated Social Security by $192 and California SDI by $1,510. The correct components are $22,599 of Social Security and $5,590 of California SDI, alongside the correctly stated Medicare amounts."
-us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model understated Social Security by using an incorrect wage cap and calculated California SDI at 1.1% instead of 1.3%. Those components are $22,599 and $5,590, respectively."
-us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The answer implies an incorrect Social Security wage-base calculation and omits most or all of the $5,590 California SDI contribution. The specified payroll-tax output includes that mandatory state employee tax in addition to all federal FICA components."
-us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced payroll-tax wage bases by employer-sponsored insurance premiums. The trace applies Social Security, Medicare, Additional Medicare Tax, and California SDI to the stated gross wages, with total wages of $430,000 for the uncapped components."
-us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model used a 1.1% California SDI rate instead of 1.3% and did not apply the correct $184,500 Social Security wage base. Correct SDI alone is $5,590 on $430,000."
-us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model deducted employer-sponsored insurance premiums from Social Security, Medicare, Additional Medicare, and SDI wage bases and used 1.1% SDI. The computation uses the full $180,000 and $250,000 gross wages and a 1.3% uncapped SDI rate."
-us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Although it listed the proper tax categories, its total understates the required components. Applying the $184,500 Social Security cap and 1.3% uncapped California SDI produces $22,599 and $5,590 for those components."
-us,scenario_005,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model used a $182,100 Social Security cap and incorrectly imposed a $165,384 California SDI wage cap at 1.2%. California SDI is uncapped at 1.3%, and the Social Security wage base is $184,500."
-us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model falsely stated that both workers were above the Social Security cap, even though the head’s $180,000 is below the $184,500 cap, and it omitted Additional Medicare Tax despite $430,000 exceeding the $250,000 joint threshold. It also omitted the $5,590 California SDI contribution."
-us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The submitted $13,905 is incompatible with the required employee taxes even though the explanation names their categories. Social Security alone is $22,599, before $6,235 of Medicare, $1,620 of Additional Medicare Tax, and $5,590 of California SDI."
-us,scenario_005,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model used a $183,600 Social Security wage base and a 1.2% California SDI rate. The correct $184,500 cap and 1.3% SDI rate raise those components to $22,599 and $5,590."
-us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model’s approximate inputs produced a total $623 too high despite identifying the correct tax categories. Exact application of the $184,500 Social Security cap and 1.3% California SDI rate yields $36,044."
-us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model understated the combined Social Security and California SDI components by $430. The exact calculation uses a $184,500 Social Security wage base and 1.3% California SDI on all $430,000."
-us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model understated the combined Social Security and California SDI components by $430. The exact calculation uses a $184,500 Social Security wage base and 1.3% California SDI on all $430,000."
-us,scenario_005,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model’s total omits or understates California SDI, which is explicitly part of the requested employee payroll-tax output. Federal employee payroll taxes total $30,454, and uncapped California SDI adds $5,590."
-us,scenario_005,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model used the wrong Social Security cap and incorrectly capped California SDI, producing only $4,226 of SDI. California SDI is uncapped at 1.3% of $430,000, or $5,590, while Social Security totals $22,599."
-us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly deducted employer health premiums from FICA and Additional Medicare wage bases and imposed an obsolete California SDI cap. The trace taxes the full $430,000 of gross wages for Medicare, Additional Medicare, and uncapped 1.3% SDI."
-us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model reduced FICA wages by employer-sponsored insurance premiums, used an obsolete Social Security cap, and capped California SDI. The computation uses gross wages of $180,000 and $250,000 and uncapped SDI of $5,590."
-us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output could not be parsed."
-us,scenario_005,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model correctly calculated the federal components but used a 1.2% California SDI rate, producing $5,160 instead of $5,590. California SDI is 1.3% of the uncapped $430,000 wage base."
-us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly treated deductions and exemptions as offsets against employee payroll taxes. Social Security, Medicare, Additional Medicare Tax, and California SDI remain due on the applicable wage bases and total $36,044."
-us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model used the obsolete $176,100 Social Security cap and incorrectly capped California SDI at $153,164 per worker while applying a 1.1% rate. The correct calculation uses a $184,500 Social Security cap and uncapped 1.3% SDI on all $430,000."
-us,scenario_005,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model’s answer effectively caps Medicare along with Social Security and omits California SDI. Medicare applies at 1.45% to all $430,000, Additional Medicare applies to $180,000, and California SDI adds $5,590."
+us,scenario_005,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a projected $183,600 Social Security wage base instead of $184,500 and a 1.2% California SDI rate instead of 1.3%. Its submitted $33,296.55 also contradicts its own recalculated $35,558.20 total."
+us,scenario_005,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly capped California SDI wages, used a 1% rate, added an employee ETT charge, and then folded $10,086.52 of state income tax into the separate payroll-tax output. California SDI is 1.3% on the full $430,000 wage base here, producing $5,590."
+us,scenario_005,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $181,800 Social Security wage base and a 1.2% California SDI rate. The applicable figures are a $184,500 Social Security base and 1.3% uncapped California SDI."
+us,scenario_005,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model correctly derived $22,599 of Social Security, $6,235 of Medicare, and $1,620 of Additional Medicare tax, but its submitted value is below even that $30,454 federal subtotal. It then excluded California SDI instead of adding the required $5,590."
+us,scenario_005,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model stated that California SDI was included but submitted approximately the federal payroll-tax subtotal alone. It failed to add the $5,590 California SDI contribution to the $30,454 federal amount."
+us,scenario_005,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $176,100 Social Security wage base and a 1.2% California SDI rate. The computation requires the $184,500 Social Security wage base and 1.3% California SDI on all $430,000 of wages."
+us,scenario_005,payroll_tax,claude-sonnet-5,llm_error,other,False,"The submitted $27,306 does not follow any of the model's own displayed totals of $29,691.40 or $34,421.40. Its working also used an obsolete Social Security wage base and a 1.1% rather than 1.3% California SDI rate."
+us,scenario_005,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model understated Social Security at $22,407 instead of $22,599 and California SDI at $4,080 instead of $5,590. It therefore used the wrong Social Security wage base and the wrong California SDI parameters."
+us,scenario_005,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model understated Social Security by using the wrong wage cap and applied a 1.1% California SDI rate. The correct components are $22,599 of Social Security and $5,590 of SDI at 1.3%."
+us,scenario_005,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The answer is consistent with calculating federal FICA and Additional Medicare tax while failing to include the full California SDI contribution. The complete calculation adds $5,590 of California SDI to the $30,454 federal subtotal."
+us,scenario_005,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced payroll wage bases by employer-sponsored insurance deductions and consequently understated the federal and California components. The trace applies Social Security, Medicare, Additional Medicare tax, and SDI to the stated $430,000 gross wage base subject only to the per-worker Social Security cap."
+us,scenario_005,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate instead of 1.3% and also understated Social Security through its wage-base choice. California SDI alone is $5,590 on $430,000."
+us,scenario_005,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model deducted employer-sponsored insurance premiums from wages before calculating Social Security, Medicare, Additional Medicare tax, and SDI. The computation uses $180,000 and $250,000 of gross employment income, and it also requires 1.3% rather than 1.1% California SDI."
+us,scenario_005,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted total understates the enumerated components by $2,230. The correct parameterization yields $22,599 of Social Security, $6,235 of Medicare, $1,620 of Additional Medicare tax, and $5,590 of California SDI."
+us,scenario_005,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly imposed a $165,384 per-worker California SDI wage cap and used a 1.2% rate. California SDI is uncapped here and equals 1.3% of all $430,000 of wages, or $5,590; it also used the wrong Social Security cap."
+us,scenario_005,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model falsely stated that both workers were above the Social Security cap even though the head's $180,000 is below the $184,500 base, and it omitted Additional Medicare tax despite $180,000 of joint wages above the $250,000 threshold. It also left out California SDI and failed to apply regular Medicare tax correctly to all wages."
+us,scenario_005,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The $13,905 answer is incompatible with the stated inclusion of Social Security, Medicare, Additional Medicare tax, and California payroll tax; regular Medicare plus Additional Medicare tax alone is $7,855. The model failed to aggregate the separately computed employee taxes across both spouses."
+us,scenario_005,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model used a projected $183,600 Social Security wage base rather than $184,500 and applied California SDI at 1.2% rather than 1.3%. Those parameter errors reduce Social Security from $22,599 and SDI from $5,590."
+us,scenario_005,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an approximate Social Security wage base and overshot the component aggregation despite correctly identifying the 1.3% California SDI rate. Applying the exact cap produces $22,599 of Social Security and a $36,044 total."
+us,scenario_005,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used estimated 2026 Social Security or California SDI parameters instead of the applicable values. The exact components total $36,044, including $22,599 of Social Security and $5,590 of SDI."
+us,scenario_005,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model's estimated California SDI and Social Security parameters understated the total. The required figures are $22,599 of Social Security and $5,590 of uncapped California SDI, alongside $7,855 of Medicare taxes."
+us,scenario_005,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"Although the model named only federal OASDI and Medicare taxes, its answer does not include the full mandatory California SDI contribution. Payroll tax requires $5,590 of California SDI in addition to the $30,454 federal subtotal."
+us,scenario_005,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly capped California SDI and produced only $4,226 rather than $5,590. SDI is 1.3% on the full uncapped $430,000 wage base; its Social Security cap was also understated."
+us,scenario_005,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"The model incorrectly applied a California SDI wage limit and a 1.2% rate. California SDI is uncapped at 1.3% here, yielding $5,590, and the exact Social Security wage base must also be used."
+us,scenario_005,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted $13,369 of employer-sponsored insurance premiums from FICA and Additional Medicare wages. It also imposed an obsolete California SDI wage cap and used 1.1%, instead of applying 1.3% to the full $430,000."
+us,scenario_005,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model reduced FICA wages by employer-sponsored insurance premiums and capped California SDI wages. The calculation uses the stated $430,000 gross wage base, with only Social Security subject to its per-worker cap, and applies uncapped 1.3% SDI."
+us,scenario_005,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so the required output was missing."
+us,scenario_005,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"The model correctly calculated the federal $30,454 subtotal but used a 1.2% California SDI rate, producing $5,160. The applicable 1.3% rate produces $5,590 and raises the total to $36,044."
+us,scenario_005,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly treated deductions and exemptions as offsets against employee payroll taxes. Social Security, regular Medicare, Additional Medicare tax, and California SDI remain due on the wage bases and total $36,044."
+us,scenario_005,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model omitted the mandatory $5,590 California SDI contribution. It also applied the employer withholding trigger of $200,000 to calculate $2,250 of Additional Medicare tax instead of the $250,000 joint-return threshold, which yields $1,620 of final liability."
+us,scenario_005,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model substituted the 2025 Social Security wage base and an obsolete capped 1.1% California SDI regime for the 2026 parameters. The correct calculation uses a $184,500 Social Security base and uncapped 1.3% SDI, yielding $22,599 and $5,590 respectively."
+us,scenario_005,payroll_tax,qwen3.8-max,llm_error,other,False,"The model incorrectly described both Social Security and Medicare taxes as capped by the Social Security wage base and failed to include California SDI. Regular Medicare applies to all $430,000, Additional Medicare tax applies to $180,000 above the joint threshold, and California SDI adds $5,590."
 us,scenario_005,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $22,800 of mortgage interest from an assumed 6.5% rate and consequently reduced taxable income to $490,509 instead of $521,425.72. It then asserted a $45,856 result inconsistent with its own stated bracket calculation of about $38,832."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted an invented $23,000 of federal income tax and treated employer health premiums and phased-out IRA contributions as additional AGI deductions. It then fabricated $21,621 of nonrefundable California credits, whereas the applicable exemption credits total only $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented approximately $60,000 of mortgage interest from an assumed interest rate, driving taxable income down to about $453,116. The trace instead allows total California deductions of $15,338.79 and yields taxable income of $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $70,000 of tax-exempt private-pension income and invented mortgage interest using an assumed 3.5% rate. Those unsupported adjustments produced a fictitious taxable-income base near $595,000 rather than the traced $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied an overstated effective California tax burden to an unsupported taxable-income estimate near $560,000. The traced rate-schedule tax is $41,208.45 on $521,425.72, followed by $156.93 of exemption credits."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model derived CA AGI as $534,270 and allowed $23,017 of itemized deductions, including $3,163 of miscellaneous employee expenses, instead of using traced AGI of $536,764.50 and deductions of $15,338.79. It also substituted estimated brackets and an estimated $500 exemption credit for the traced $156.93 credit."
-us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented $50,000 of mortgage interest by assuming a 5% rate and deducted it despite the prompt supplying no mortgage-interest payment. This reduced its taxable income to $464,416 instead of $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $497,700 rather than the traced $521,425.72. It also subtracted an approximate $300 exemption credit instead of the applicable $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model claimed $73,017 of itemized deductions by including mortgage interest not supplied in the facts. The traced California deduction is $15,338.79, producing taxable income of $521,425.72 rather than $461,253."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer supplies no usable derivation, and $31,215 does not result from the traced taxable income, rate schedule, and credits. Applying the schedule to $521,425.72 gives $41,208.45 before subtracting $156.93 of exemption credits."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used taxable income of roughly $501,848 instead of $521,425.72. It therefore understated the 9.3% bracket base before applying the nonrefundable exemption credits."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used CA AGI of $520,901 and deductions of $19,854, yielding a tax base near $501,047. The traced inputs are AGI of $536,764.50 and deductions of $15,338.79, yielding $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the listed property tax, charity, and employee expenses as the controlling California itemized deductions without applying the traced deduction limitations. California deductions are $15,338.79, and the resulting schedule tax is $41,208.45 before $156.93 of credits, not $42,150."
-us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The generic explanation never states its income base, deduction amount, brackets, or credits. The submitted $36,918 omits part of the liability generated by $521,425.72 of taxable income under the California schedule."
-us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $77,920 of qualified dividends from AGI and incorrectly asserted a $2,000 California capital-loss limit. Those errors drove CA AGI to $449,439 instead of $536,764.50 and taxable income far below $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no numerical derivation and its $20,677 result is consistent with applying a much lower effective rate or income base than the California schedule requires. Tax on the traced $521,425.72 is $41,208.45 before the $156.93 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model supplied only a generic list of considerations and no computation supporting $37,099. The traced taxable income and California schedule yield $41,051.51 after nonrefundable credits."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model says it reduced wages for both traditional 401(k) deferrals and pretax health premiums, thereby understating the California income base. The traced calculation instead reaches AGI of $536,764.50 and taxable income of $521,425.72."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $519,427 of taxable income, $1,998.72 below the traced $521,425.72. That understated 9.3% bracket income, with its treatment of exemption credits accounting for the remaining offset in its near-miss."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model's unexplained estimate exceeds the traced result despite naming the relevant components. The exact schedule produces $41,208.45, and the exact nonrefundable exemption credit is $156.93, yielding $41,051.51 rather than $41,150."
-us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly says California taxes the listed tax-exempt interest and does not provide a reconciled AGI or deduction calculation. The controlling figures are $536,764.50 of AGI and $15,338.79 of deductions, not the unstated base behind $40,166."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model provides no calculation and its $48,720 answer overstates the liability generated by the California rate schedule. The schedule yields $41,208.45 before subtracting $156.93 of nonrefundable credits."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $534,270 and overstated deductions enough to produce taxable income near $512,000 rather than $521,425.72. It also used approximate tax and credit amounts instead of $41,208.45 and $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used CA AGI of $525,049 and deductions of $23,201, producing taxable income of $501,848. The trace uses $536,764.50 of AGI and $15,338.79 of deductions, and the model additionally failed to subtract the $156.93 exemption credit."
-us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added the $70,000 tax-exempt private pension and $8,961 tax-exempt interest to California AGI, inflating it to about $604,010. The traced CA AGI is $536,764.50, leading to taxable income of $521,425.72 rather than $582,388."
-us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. This is a missing-output contract failure rather than a tax computation.
-us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $534,270 and used $21,837 of deductions, yielding taxable income of $512,433 rather than $521,425.72. It also used a $258 exemption credit instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the loan balance and concluded that deductions eliminated taxable income despite more than $500,000 of traced taxable income. It also invoked the federal $10,000 SALT cap in a California deduction calculation, while the trace applies the California limitation to the $17,280 property-tax input."
-us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $46,308 of traditional 401(k) deferrals and failed to apply the $3,000 net-capital-loss limit when constructing income. It then used only a standard deduction and overstated personal exemption credits as $892 instead of $156.93."
-us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's stated taxable income of about $495,603 is below the traced $521,425.72, and its $29,423 tax is also inconsistent with California's progressive schedule even on its own stated base. The correct schedule tax is $41,208.45 before $156.93 of exemption credits."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented roughly $22,800 of mortgage interest even though no mortgage-interest payment was listed, then used a $43,761 itemized deduction instead of the allowable $15,338.79. It also failed to reconcile its stated $38,832 bracket calculation with its submitted $45,856."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted an invented $23,000 of federal income tax and treated employer premiums and phased-out IRA contributions as additional reductions. It then fabricated $21,621 of nonrefundable California credits instead of subtracting the $156.93 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model inferred about $60,000 of mortgage interest from the loan balance despite the instruction that unlisted payments are zero. That fictitious deduction reduced taxable income to $453,116 instead of $521,425.72."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $70,000 tax-exempt private pension to California income and invented mortgage interest from an assumed rate. Those unsupported inputs displaced the traced AGI of $536,764.50 and deduction of $15,338.79."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used a rough $560,000 taxable-income estimate and invoked rates up to 11.3%, although $521,425.72 remains in California's 9.3% bracket for joint filers. The schedule yields $41,208.45 before the $156.93 exemption credit, not $47,500."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model computed California AGI as $534,270 and deductions as $23,017, rather than the traced $536,764.50 AGI and $15,338.79 deduction. It also used estimated bracket boundaries and a $500 exemption credit instead of the applicable $156.93 credit."
+us,scenario_005,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented $50,000 of mortgage interest from an assumed 5% rate and deducted it despite no mortgage-interest amount being listed. This reduced its taxable income to $464,416 instead of $521,425.72."
+us,scenario_005,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used $497,700 of taxable income rather than $521,425.72 and supplied no deductions supporting that reduction. The proper schedule and $156.93 exemption credit produce $41,051.51."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included invented mortgage interest in a $73,017 itemized deduction. The allowable California deduction is $15,338.79, producing taxable income of $521,425.72 rather than $461,253."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $31,215 reflects a substantially understated California tax base or schedule tax. The traced taxable income is $521,425.72, on which tax before credits is $41,208.45 and the only applied nonrefundable credits total $156.93."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used taxable income of roughly $501,848 instead of $521,425.72. It therefore subtracted about $19,578 too much through its unspecified California adjustments or deductions."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated California AGI at $520,901 and overstated deductions at $19,854. The traced figures are $536,764.50 of AGI and a $15,338.79 deduction, yielding $521,425.72 of taxable income."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $42,150 does not apply the California joint-filer schedule and exemption credit to the traced base. Tax on $521,425.72 is $41,208.45 before subtracting $156.93 of nonrefundable credits."
+us,scenario_005,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The submitted $36,918 implies that the model reduced the California tax base below the traced $521,425.72 through excessive pretax adjustments or deductions. California tax before credits is $41,208.45 on that base, followed by only $156.93 of nonrefundable credits."
+us,scenario_005,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted qualified dividends from AGI, producing federal AGI of $452,587 and California AGI of $449,439. It also incorrectly used a $2,000 California capital-loss limit; the traced California AGI is $536,764.50."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $20,677 is consistent with applying neither the full joint-filer schedule nor the correct tax base. Taxable income is $521,425.72, producing $41,208.45 before the $156.93 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted $37,099 implies excessive retirement, itemized-deduction, or credit reductions. The correct taxable income is $521,425.72 and only $156.93 of nonrefundable exemption credits reduces the $41,208.45 schedule tax."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly treated pretax health premiums as additional reductions from the listed wages and understated the California tax base. The traced calculation uses $536,764.50 of AGI and $15,338.79 of deductions."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $519,427 of taxable income instead of $521,425.72. Applying the schedule to the traced base and subtracting $156.93 gives $41,051.51."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model's aggregate estimate did not apply the exact 2026 California schedule and $156.93 exemption credit. Those steps convert $521,425.72 of taxable income into $41,051.51, not $41,150."
+us,scenario_005,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model states that California taxes the listed tax-exempt interest, which misstates the income adjustment used in this case. The traced California AGI is $536,764.50, and the resulting liability after the $156.93 exemption credit is $41,051.51."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The unexplained $48,720 substantially overstates the schedule tax on $521,425.72. California's joint-filer schedule produces $41,208.45 before subtracting $156.93 of nonrefundable credits."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $534,270 and used roughly $21,837 of deductions, producing taxable income near $512,000. The traced figures are $536,764.50 of AGI, $15,338.79 of deductions, and $521,425.72 of taxable income."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used $538,418 as California AGI and $22,934 of itemized deductions. The engine calculation instead uses $536,764.50 of AGI and a $15,338.79 deduction, then subtracts a $156.93 exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated California AGI at $525,049 and overstated itemized deductions at $23,201, yielding taxable income of only $501,848. It also failed to subtract the applicable $156.93 nonrefundable exemption credit."
+us,scenario_005,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added the $70,000 tax-exempt private pension and $8,961 tax-exempt interest to California AGI, inflating it to about $604,010. The traced California AGI is $536,764.50."
+us,scenario_005,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required numeric result was absent."
+us,scenario_005,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used California AGI of $534,270 and deductions of about $21,837, producing taxable income of $512,433. The traced calculation uses $536,764.50 of AGI and a $15,338.79 deduction, yielding $521,425.72."
+us,scenario_005,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the loan balance, applied a federal $10,000 SALT cap to the California deduction, and concluded that deductions eliminated all tax despite more than $500,000 of taxable income. The correct California taxable income is $521,425.72."
+us,scenario_005,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income at about $526,500 by using only about $11,912 of deductions. The allowable deduction is $15,338.79, producing taxable income of $521,425.72 and tax after credits of $41,051.51."
+us,scenario_005,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the two $23,154 traditional 401(k) deferrals and failed to enforce the $3,000 net-capital-loss limit consistently, yet reported a tax far below its own $565,375 taxable-income estimate. It also used an unphased $892 exemption credit instead of the applicable $156.93."
+us,scenario_005,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an unsupported $495,603 taxable-income estimate and then understated the California schedule tax even on that amount. The traced taxable income is $521,425.72, with $41,208.45 of schedule tax before the $156.93 exemption credit."
 us,scenario_005,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model discarded all SSDI from taxable income after correctly recognizing that Social Security taxability depends on combined income. The combined-income worksheet makes $13,784.12 taxable, so AGI is $49,064.12 rather than $35,280."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived $13,784 of taxable Social Security, AGI near $49,064, and tax near $3,708, then replaced that computation with an unsupported $2,386 submission. Nothing in its stated calculation produces the submitted value."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated 85% of total benefits as taxable instead of applying the two-tier Social Security worksheet, which yields $13,784.12. It then submitted $1,858 despite its own overstated-income calculation producing about $4,021."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income of $32,964 and tax of about $3,708, but submitted $2,534. The final value abandons its own correct Social Security, deduction, and bracket computation."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model equated being above the upper combined-income threshold with taxing a full 85% of SSDI. The worksheet instead limits taxable Social Security to $13,784.12, and the model also used a $15,000 proxy rather than the 2026 $16,100 standard deduction."
-us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used an unsupported worksheet approximation for taxable Social Security and then submitted $1,611 even though its own bracket calculation produced about $3,941. The submitted value does not follow from any stated deduction, rate, or credit."
-us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,300 standard deduction instead of the 2026 single deduction of $16,100. That overstated taxable income by $800 and consequently overstated regular income tax."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly found taxable Social Security and AGI but used an estimated $15,700 standard deduction instead of $16,100. It also used obsolete estimated bracket endpoints rather than the applicable 2026 main-rate parameters."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model treated the 56-year-old single taxpayer as a head-of-household filer over age 65. The taxpayer is under 65 and has no qualifying dependent, so the single filing status and $16,100 deduction apply, with no age-based addition."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied a pre-TCJA personal exemption in addition to its deduction. The 2026 computation uses the $16,100 single standard deduction with no personal exemption, producing taxable income of $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model substituted a small $8,300 standard deduction and a $5,150 personal exemption for the applicable $16,100 single standard deduction. This left taxable income too high and produced the wrong regular tax."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI of $49,064.13 but implicitly used the wrong standard deduction or bracket parameters. The exact $16,100 deduction leaves $32,964.12 taxable, whose 2026 main-rate tax is $3,707.70 rather than $3,807.18."
-us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The explanation explicitly derives the correct $3,707.70 tax, but the model submitted $3,744. The numeric output contradicts its own final calculation."
-us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2024's $14,600 standard deduction and 2024 brackets instead of the 2026 parameters. It also calculated taxable Social Security as $14,025.53 rather than $13,784.12 by misapplying the upper-tier worksheet."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction eliminated the taxable pension and taxable Social Security. AGI is $49,064.12, and subtracting $16,100 still leaves $32,964.12 of taxable income."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model zeroed the liability despite $35,280 of taxable pension and $13,784.12 of taxable Social Security. No listed nonrefundable credit offsets the $3,707.70 regular tax."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,750 standard deduction rather than the exact 2026 amount of $16,100. That inflated taxable income and led to a rounded tax above $3,707.70."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model asserted taxable income of about $35,572, which corresponds to overstating taxable Social Security near the 85% cap. The correct Social Security worksheet and $16,100 deduction produce taxable income of $32,964.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income at $33,206 instead of deriving $32,964.12 from $49,064.12 of AGI and the $16,100 deduction. Its approximate deduction or Social Security calculation therefore overstated the tax."
-us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated the correct calculation structure but did not use the exact 2026 deduction and main-rate parameters. Those parameters tax $32,964.12 at $3,707.70, not $3,736.66."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $2,444 is consistent with omitting most or all of the $13,784.12 taxable Social Security from AGI. Including it and subtracting the $16,100 standard deduction leaves $32,964.12 taxable."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly projected a post-TCJA-sunset regime with a personal exemption and 15% bracket. The applicable 2026 computation uses the $16,100 standard deduction and the 2026 main brackets, with no personal exemption."
-us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed the full 85% cap of SSDI, $16,392, instead of the worksheet amount of $13,784.12. It also used an estimated $15,420 deduction rather than $16,100, overstating taxable income on both steps."
-us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. Its response therefore failed the required output contract before any substantive tax calculation could be evaluated.
-us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly identified AGI of $49,064.13 but applied an unspecified deduction or exemption that left taxable income too high. The only applicable basic reduction is the $16,100 single standard deduction, after which the tax is $3,707.70."
-us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security entirely and treated AGI as only the $35,280 pension. The combined-income worksheet adds $13,784.12 of taxable SSDI, raising AGI to $49,064.12."
-us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model misapplied the Social Security worksheet by using $6,000 in the upper-tier formula and obtained $15,525.53 instead of $13,784.12. It also used an estimated $15,750 standard deduction rather than $16,100."
-us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model's explanation computes tax of $2,177 from pension-only taxable income but submits $4,202, so the numeric answer does not follow from its stated reasoning. It also omits the $13,784.12 taxable portion of SSDI from the displayed taxable-income calculation and uses a 12.5% rate instead of the applicable 12% bracket."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model ultimately treated all $19,285 of SSDI as nontaxable despite combined income requiring $13,784.12 to be included. It also used a $14,600 standard deduction instead of the 2026 amount of $16,100."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived taxable Social Security, AGI, and taxable income closely enough to calculate about $3,708, then submitted $2,386 without any supporting computation. Its final value contradicts its own tax-bracket calculation."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly capped taxable Social Security at $16,392.25 instead of applying the worksheet result of $13,784.12. It then submitted $1,858 even though its own overstated-income calculation produced about $4,021."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived $13,784 of taxable Social Security, $32,964 of taxable income, and approximately $3,708 of tax. It nevertheless submitted $2,534, a value unsupported by and inconsistent with its stated computation."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated 85% of the entire SSDI benefit as taxable merely because combined income exceeded $34,000. The Social Security worksheet instead limits taxable benefits to $13,784.12, and the model also substituted 2025 estimates for the 2026 $16,100 standard deduction and applicable brackets."
+us,scenario_007,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used an unsupported $15,650 taxable-Social-Security approximation instead of $13,784.12 and calculated tax near $3,941. It then submitted $1,611, which does not follow from any figures in its reasoning."
+us,scenario_007,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,300 standard deduction rather than the 2026 single deduction of $16,100. This overstated taxable income by $800 and therefore overstated regular income tax."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included about $13,784 of taxable Social Security but used an estimated $15,700 standard deduction instead of $16,100. It also used obsolete bracket parameters rather than the applicable 2026 main-rate brackets."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model treated this 56-year-old single person as a head-of-household filer over age 65. That invented filing status and age-based deduction displaced the required single-filer calculation using the $16,100 standard deduction."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied pre-TCJA personal-exemption rules to 2026 instead of the applicable 2026 single standard deduction and main-rate brackets. The required taxable income is $32,964.12 and its regular tax is $3,707.70."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model replaced the 2026 $16,100 single standard deduction with an $8,300 deduction plus a $5,150 personal exemption. Those obsolete deduction assumptions overstated taxable income and tax."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI of about $49,064 but implicitly used the wrong standard deduction or bracket parameters. Subtracting $16,100 yields $32,964.12 of taxable income, whose 2026 main-rate tax is $3,707.70, not $3,807.18."
+us,scenario_007,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The explanation explicitly computes the requested value as $3,707.70, but the submitted numeric answer was $3,744. The error is a final-output mismatch rather than a tax-rule error."
+us,scenario_007,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2024's $14,600 standard deduction and 2024 brackets for a 2026 calculation. It also calculated taxable Social Security as $14,025.53 rather than the worksheet amount of $13,784.12."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the pension and taxable Social Security as fully offset by the standard deduction. They instead produce $49,064.12 of AGI and $32,964.12 of taxable income, leaving $3,707.70 of tax."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the liability to zero through unspecified deductions or credits. The $16,100 standard deduction leaves $32,964.12 taxable, and no nonrefundable credits reduce the resulting $3,707.70 tax."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI near $49,064 but used an estimated $15,750 standard deduction rather than $16,100. It then rounded away the remaining bracket-calculation difference instead of applying the exact 2026 parameters."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $35,572 rather than $32,964.12. That figure reflects overstating taxable Social Security to 85% of benefits instead of applying the worksheet result of $13,784.12."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income at $33,206 instead of deriving $32,964.12 from $49,064.12 of AGI less the $16,100 deduction. This incorrect taxable-income estimate carried into its regular-tax result."
+us,scenario_007,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated the correct components but did not apply the exact 2026 deduction and main-rate bracket parameters. Those parameters yield taxable income of $32,964.12 and tax of $3,707.70, not $3,736.66."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's $2,444 result omits a substantial portion of the $13,784.12 taxable Social Security inclusion or applies an unsupported deduction. Correct AGI is $49,064.12, leaving $32,964.12 taxable after the standard deduction."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA sunset and used a reduced standard deduction, personal exemption, and 15% bracket. The applicable 2026 calculation instead uses the $16,100 standard deduction and current main-rate brackets."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly substituted projected post-TCJA rules featuring a personal exemption and a 15% bracket. Applying the actual 2026 $16,100 standard deduction and main-rate brackets produces $3,707.70."
+us,scenario_007,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model taxed 85% of all SSDI, producing $16,392 of taxable benefits, instead of applying the worksheet result of $13,784.12. It also used an estimated $15,420 standard deduction rather than $16,100."
+us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. Its response therefore failed the required output contract.
+us,scenario_007,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly identified AGI of $49,064.13 but applied an unspecified and incorrect deduction/exemption or rate schedule. The 2026 $16,100 standard deduction leaves $32,964.12 taxable, and the applicable brackets yield $3,707.70."
+us,scenario_007,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security entirely and used only the $35,280 pension in AGI. The Social Security worksheet adds $13,784.12, raising AGI to $49,064.12 and taxable income to $32,964.12."
+us,scenario_007,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly calculated taxable Social Security as $14,196 after including the $284 tax-exempt pension in combined income. The traced worksheet yields $13,784.12, so AGI is $49,064.12 rather than $49,476."
+us,scenario_007,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model misapplied the Social Security worksheet by using $6,000 in the upper-tier formula, producing $15,525.53 taxable benefits instead of $13,784.12. It also used an estimated $15,750 standard deduction rather than $16,100."
+us,scenario_007,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model's reasoning omits taxable Social Security and computes only $2,177 of tax from pension income, yet it submits $4,202. The submitted value is unsupported by its calculation, and the required derivation includes $13,784.12 of taxable Social Security and yields $3,707.70."
 us,scenario_007,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_007,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model explicitly assumed that the 24-month SSDI waiting period was satisfied from the constant-full-year-status instruction. That instruction establishes no within-year change, not 24 months of prior SSDI entitlement, so the 56-year-old has no established Medicare eligibility pathway."
 us,scenario_007,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model converted a year-long constant SSDI status into completion of Medicare’s 24-month disability waiting period. Full-year receipt does not establish the required prior entitlement duration, and age 56 does not satisfy the age pathway."
@@ -260,117 +272,126 @@ us,scenario_007,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,Fals
 us,scenario_007,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model first recognized that no wage was listed but then asserted $1,516.30 of employee Social Security and Medicare tax on unspecified “applicable taxable wages.” The listed Social Security disability and private pension payments are not employee FICA wages, and the 50 weekly hours cannot create wage income when the unlisted wage rate and annual wages are zero."
 us,scenario_007,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract."
 us,scenario_007,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached $19,180 of Idaho taxable income but abandoned that derivation and submitted $1,571 without a supporting calculation. Applying the applicable Idaho brackets and rates to $19,180 yields $755.78, not the unsupported inflation adjustment it asserted."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated the entire $35,280 private pension as qualifying for an Idaho retirement-benefits deduction. This 56-year-old's pension is not fully excluded under that pathway, so $19,180 remains taxable after the Social Security subtraction and standard deduction."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly derived approximately $19,180 of Idaho taxable income, then replaced Idaho's applicable bracket computation with a 5.695% flat-rate estimate and finally submitted an unexplained still-higher amount. The bracket calculation on $19,180 produces $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded all SSDI from federal AGI and used an estimated $15,750 standard deduction plus an invented separate threshold. Federal AGI includes $13,784.12 of taxable Social Security, Idaho subtracts that amount, and the $16,100 standard deduction leaves $19,180 for the bracket calculation."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached the correct $19,180 taxable-income base but applied a 5.3% flat-rate-plus-exempt-amount shortcut that does not represent the applicable Idaho bracket calculation. That calculation yields $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model taxed 85% of SSDI outright instead of using the Social Security inclusion formula, then denied Idaho's subtraction for the federally taxable Social Security amount and invented a personal exemption. Idaho subtracts $13,784.12, applies only the stated $16,100 standard deduction here, and taxes $19,180 under its brackets."
-us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an estimated standard deduction and then increased its own $1,178 flat-rate calculation to $1,800 for unspecified add-backs and phase-ins. Idaho taxable income is $19,180, and the applicable brackets and rates yield $755.78 without those invented adjustments."
-us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction instead of $16,100 and taxed the resulting base at a flat 5.8%. The correct taxable income is $19,180, to which Idaho's applicable brackets and rates produce $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction rather than $16,100 and imposed a flat 5.8% rate. The correct $19,180 taxable-income base must be run through Idaho's applicable brackets and rates, yielding $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer's reference to taxable disability income implies that it retained SSDI in the Idaho tax base instead of subtracting the $13,784.12 federally taxable portion. After that subtraction and the $16,100 standard deduction, only $19,180 is subject to Idaho's bracket calculation."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Although the model recognized that Idaho excludes Social Security, its $1,246 answer applies the wrong effective rate or taxable-income base to the remaining pension income. The $16,100 standard deduction leaves $19,180, and Idaho's brackets and rates yield $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model produced $21,830 of taxable income by using the wrong deduction structure, including a personal-exemption treatment absent from the trace, and then applied a flat 5.695% rate. The correct standard deduction is $16,100, leaving $19,180 for the Idaho bracket calculation."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly subtracted $13,784.13 of taxable Social Security but used the wrong standard deduction or rate to reach $1,115.08. A $16,100 standard deduction leaves $19,180, and the applicable Idaho brackets and rates produce $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The reported answer is $1,894, while its explanation ends by submitting $1,092.30, violating the required agreement between the numeric value and explanation. Its stated $1,092.30 calculation also uses an incorrect flat-rate treatment instead of the brackets that yield $755.78 on $19,180."
-us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model invented a disabled-person retirement-benefit deduction that fully excluded the $35,280 private pension. The pension remains in Idaho AGI, and after the $16,100 standard deduction the taxable amount is $19,180."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exclusions and the standard deduction eliminate all taxable income. Idaho AGI is $35,280 after the Social Security subtraction, so the $16,100 standard deduction leaves $19,180 taxable."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero answer implies that the model erased the $35,280 taxable pension through unsupported deductions or credits. After the Social Security subtraction and $16,100 standard deduction, $19,180 remains taxable and produces $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly reached pension-only Idaho AGI and used the $16,100 standard deduction, but then invented a separate $5,000 zero bracket and applied 5.3% only above it. Idaho's actual bracket computation on $19,180 yields $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The unexplained $1,885 answer implies a substantially overstated taxable base or rate and omits the decisive Idaho Social Security subtraction and $16,100 standard deduction. Those steps leave $19,180, whose bracket tax is $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly computed $19,180 of taxable income but applied an approximately 5.3% flat rate. Idaho's applicable bracket calculation on that base yields $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The answer applied a federal-based taxable-income shortcut without accounting correctly for Idaho's $13,784.12 Social Security subtraction and $16,100 standard deduction. Those adjustments leave $19,180, not the larger base implied by $1,759.89."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model gave no calculation supporting $1,477 and its number is inconsistent with the Idaho bracket tax on pension income after the correct deduction. The $16,100 standard deduction leaves $19,180, producing $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,535 standard deduction instead of $16,100, inflating taxable income from $19,180 to $26,745. It then compounded the error by applying a flat 5.695% rate rather than the applicable brackets."
-us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $36,252 taxable-income figure retains federally taxable SSDI in the Idaho base instead of subtracting $13,784.12. It also used the wrong standard deduction and a flat 5.8% rate; the correct taxable base is $19,180."
-us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used a $32,964 federal-taxable-income base instead of deriving Idaho AGI after the Social Security subtraction and then applying the $16,100 standard deduction. The correct Idaho taxable income is $19,180, not $32,964."
-us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required output contract before any substantive tax calculation could be evaluated.
-us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used the wrong deduction structure to get $21,680, then added a $2,500 zero bracket and applied a flat 5.3% rate. The $16,100 standard deduction leaves $19,180, and Idaho's applicable brackets produce $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Idaho has no individual income tax. Idaho taxes the household's $19,180 of taxable income under its applicable brackets, producing $755.78."
-us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $19,285 of SSDI as federally and state taxable, failed to apply Idaho's $13,784.12 subtraction, and used the wrong standard deduction. Its later unexplained credits do not repair the inflated base; the correct Idaho taxable income is $19,180."
-us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted an additional $5,200 Idaho standard deduction from a base that already reflected a standard deduction, double-counting deductions and reducing taxable income to $14,330. The correct single standard deduction is applied once, leaving $19,180 for the applicable bracket calculation."
-us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model applied an obsolete $120 Grocery Credit amount instead of Idaho's 2026 base amount of $155. The resident's age does not reduce the applicable 2026 base credit to $120.
-us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that Idaho has no refundable individual income-tax credit. The full-year Idaho resident qualifies for the refundable Grocery Credit of $155.
-us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used the wrong Grocery Credit schedule, describing a $120 amount and then submitting an unsupported $140. The applicable 2026 base amount for this one-person household is $155."
-us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly treated the Grocery Credit as nonrefundable when claimed on the income-tax return and excluded it from the requested output. This qualifying full-year resident receives the $155 credit as a refundable state credit.
-us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model misclassified Idaho's Grocery Credit as a nonrefundable or other item. It belongs in refundable state credits and contributes $155 for this resident.
-us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly made refundability depend on whether the taxpayer had remaining tax liability and treated the credit as nonrefundable when used against that liability. The Grocery Credit is refundable, and neither taxable pension income nor SSDI moves its $155 amount out of this output."
-us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model incorrectly classified Idaho's Grocery Credit as nonrefundable in this calculation. The household qualifies for a $155 refundable Grocery Credit.
-us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,The model identified the correct credit but used an obsolete $120 amount. Idaho's 2026 Grocery Credit base amount for this qualifying person is $155.
-us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used $120 per household member instead of the 2026 Grocery Credit base amount of $155. One qualifying resident therefore produces $155, not $120."
-us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model denied eligibility without applying Idaho's Grocery Credit rules. The head qualifies for all 12 months and receives the refundable $155 base credit.
-us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified Grocery Credit eligibility but applied an obsolete $120 amount. The applicable 2026 amount is $155 for this qualifying individual.
-us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used a $120 Grocery Credit parameter for a resident under age 65. The 2026 base credit awarded to this resident is $155.
-us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model applied an obsolete $120 per-resident Grocery Credit. The applicable 2026 base amount is $155.
-us,scenario_007,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model recognized the refundable Grocery Credit but used the wrong amount. The 2026 credit for this qualifying resident is $155 rather than $120.
-us,scenario_007,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model falsely treated Idaho's Grocery Credit as repealed. It remains applicable in 2026 and awards this qualifying full-year resident a refundable $155 credit.
-us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to apply the Idaho Grocery Credit to the full-year resident. That refundable credit contributes $155 even though no children are listed.
-us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model overlooked that Idaho residence for the full tax year supplies the relevant qualifying condition for the Grocery Credit. Applying the program yields a refundable credit of $155.
-us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model invented an income phaseout that eliminates the Idaho Grocery Credit for this household. The household remains eligible for the full 2026 base credit of $155.
-us,scenario_007,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model failed to recognize the refundable Idaho Grocery Credit as applicable. The qualifying resident receives $155.
-us,scenario_007,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model used an obsolete $100 Grocery Credit amount. The applicable 2026 base amount for the full-year resident is $155.
-us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model omitted the Idaho Grocery Credit from applicable refundable credits. This full-year resident qualifies for its $155 base amount.
-us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required output contract rather than performing the Idaho Grocery Credit calculation.
-us,scenario_007,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly identified Grocery Credit eligibility but used the obsolete $120 amount. The 2026 amount for one qualifying person is $155.
-us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly asserted that Idaho has no state income tax and consequently excluded all state refundable credits. Idaho's refundable Grocery Credit awards this resident $155.
-us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model overlooked the Idaho Grocery Credit applicable to this household. The full-year qualifying resident receives a refundable $155 credit.
-us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly concluded that no Idaho refundable credit applies. The household qualifies for the $155 refundable Grocery Credit for the full year.
-us,scenario_008,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only CHIP's age and upper-income tests and omitted the requirement that a CHIP child not qualify for Medicaid. Child1 qualifies for Medicaid in New Jersey's OLDER_CHILD category, so CHIP is precluded despite income being below the CHIP ceiling."
-us,scenario_008,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being under 19 and below New Jersey's CHIP income threshold as sufficient for eligibility. It failed to test Medicaid first: child1 qualifies in the OLDER_CHILD category, which makes child1 ineligible for CHIP."
-us,scenario_008,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly made CHIP eligibility depend only on age, household income, and lack of listed coverage. The controlling exclusion is Medicaid eligibility, and child1 qualifies for Medicaid as an OLDER_CHILD, so CHIP eligibility is false regardless of being under the CHIP income ceiling."
-us,scenario_008,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child is CHIP-eligible without checking the Medicaid pathway. Child1 is Medicaid-eligible under New Jersey's OLDER_CHILD category, and Medicaid eligibility bars CHIP."
-us,scenario_008,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model treated the under-19 age rule and income below New Jersey's CHIP ceiling as jointly sufficient. It omitted the Medicaid-exclusion step: child1 qualifies for Medicaid as an OLDER_CHILD and therefore cannot qualify for CHIP.
-us,scenario_008,child1_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from age 17 and low household income. It failed to determine that child1 is Medicaid-eligible in the OLDER_CHILD category, which automatically precludes CHIP."
-us,scenario_008,child1_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model compared household MAGI with the CHIP income limit but did not apply the requirement that the child be ineligible for Medicaid. Child1 qualifies for Medicaid under New Jersey's OLDER_CHILD category, so CHIP eligibility is false."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached $19,180 of Idaho taxable income, then abandoned that derivation and substituted an unsupported $1,571 estimate. It failed to apply the 2026 Idaho schedule: subtract the $4,920 zero-rate threshold and tax the remaining $14,260 at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the entire $35,280 private pension as qualifying for Idaho’s retirement-benefits deduction. This 56-year-old does not qualify for a deduction that eliminates the pension, so taxable income remains $19,180 after the Social Security subtraction and standard deduction."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly derived approximately $19,180 of taxable income but replaced the schedule calculation with an unsupported $1,379 estimate. The correct step is ($19,180 - $4,920) × 5.3%, not a higher estimated rate or conformity adjustment."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded all Social Security disability income from federal AGI and used an estimated standard deduction instead of the $16,100 amount. It also failed to carry its own computed tax through to the submitted value; the correct Idaho base is $19,180 and the schedule yields $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model identified the $19,180 taxable-income base and the existence of a zero-rate threshold but computed the schedule incorrectly. Subtracting the exact $4,920 threshold and applying 5.3% gives $755.78, not $1,112."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $13,784.12 of federally taxable Social Security benefits from Idaho AGI and instead invented a personal exemption and grocery-credit treatment in the tax base. Idaho AGI is $35,280 after the Social Security subtraction, taxable income is $19,180, and the bracket calculation produces $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model estimated the standard deduction and then added unspecified adjustments to inflate a calculation it had already placed near $1,178. It needed to use the $16,100 deduction and Idaho’s $4,920 zero-rate threshold, leaving $14,260 taxed at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $15,300 standard deduction instead of $16,100 and applied 5.8% to the entire taxable-income base. The 2026 calculation uses $19,180 of taxable income, removes the $4,920 zero-rate portion, and applies 5.3% to $14,260."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and treated Idaho tax as 5.8% of all taxable income. The applicable deduction is $16,100, followed by the $4,920 zero-rate threshold and a 5.3% rate on the remainder."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer’s unexplained $1,582 is inconsistent with the traced $19,180 taxable-income base. It reflects taxing an inflated base or omitting the Social Security subtraction, standard deduction, or zero-rate threshold instead of applying the Idaho schedule to $19,180."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Although the model recognized that Idaho excludes Social Security benefits, its $1,246 answer does not apply the 2026 schedule to the resulting $19,180 taxable income. The first $4,920 is taxed at zero and only $14,260 is taxed at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model derived $21,830 by using the wrong deduction and personal-exemption treatment. The Idaho Social Security subtraction leaves $35,280, the $16,100 standard deduction leaves $19,180, and the bracket threshold then reduces the taxed amount to $14,260."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly began with federal AGI and the Idaho Social Security subtraction but did not apply the exact $16,100 deduction and $4,920 zero-rate threshold. The resulting taxable amount subject to 5.3% is $14,260, producing $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model’s stated $1,092.30 calculation taxes essentially all $19,180 at an outdated higher rate and its submitted $1,894 conflicts with that explanation. Idaho’s schedule excludes the first $4,920 and taxes the remaining $14,260 at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly granted a disability-based retirement-benefits deduction that fully eliminated the $35,280 private pension. The pension remains in Idaho AGI, leaving $19,180 after the standard deduction and a positive $755.78 liability under the bracket schedule."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exclusions and the standard deduction fully offset the income. After subtracting taxable Social Security from federal AGI, $35,280 remains, and the $16,100 standard deduction leaves $19,180 of Idaho taxable income."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero answer incorrectly implies that deductions or nonrefundable credits eliminate the liability. The traced deductions leave $19,180 of taxable income, of which $14,260 is taxed at 5.3%, producing $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $5,000 zero-rate threshold instead of Idaho’s $4,920 threshold. With the otherwise correct $19,180 taxable-income base and 5.3% rate, the exact calculation is ($19,180 - $4,920) × 0.053 = $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The unsupported $1,885 answer implies taxation of an inflated base or failure to apply Idaho’s zero-rate bracket. The correct derivation leaves $19,180 taxable, subtracts the $4,920 threshold, and taxes $14,260 at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly obtained $19,180 of taxable income but applied 5.3% to the entire amount. Idaho’s first $4,920 is in the zero-rate bracket, so only $14,260 is taxed and the liability is $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $1,759.89 answer reflects an inflated federal-based taxable base and does not incorporate the full Idaho Social Security subtraction and bracket threshold. Those steps leave $19,180 of taxable income and $14,260 subject to the 5.3% rate."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model’s $1,477 estimate does not follow Idaho’s 2026 schedule after the pension income and standard deduction. Taxable income is $19,180, the first $4,920 is taxed at zero, and 5.3% applies only to the remaining $14,260."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $8,535 standard deduction, inflating taxable income to $26,745. The correct standard deduction is $16,100, leaving $19,180 before application of the $4,920 zero-rate threshold."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model’s $26,803 taxable-income figure shows that it used the wrong conforming standard deduction. Subtracting the correct $16,100 from the $35,280 Idaho AGI gives $19,180, after which the zero-rate threshold and 5.3% rate yield $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model retained taxable Social Security in the state base, producing $36,252 after its estimated deduction. Idaho subtracts $13,784.12 of taxable Social Security first and uses a $16,100 standard deduction, leaving $19,180 rather than $36,252."
+us,scenario_007,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $32,964 federal taxable-income base instead of performing Idaho’s Social Security subtraction and $16,100 standard deduction. Those operations produce $19,180, and the subsequent zero-rate threshold reduces the amount taxed at 5.3% to $14,260."
+us,scenario_007,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required structured answer could not be parsed."
+us,scenario_007,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used the wrong $21,680 taxable-income figure and a $2,500 zero-rate threshold. The correct figures are $19,180 after the $16,100 standard deduction and a $4,920 zero-rate threshold, leaving $14,260 taxed at 5.3%."
+us,scenario_007,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Idaho has no individual income tax. Idaho imposes individual income tax, and this household owes $755.78 after the state subtraction, standard deduction, and bracket calculation."
+us,scenario_007,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly reached $19,180 but applied 5.3% to the whole amount. It omitted Idaho’s $4,920 zero-rate threshold, which leaves $14,260 subject to tax and produces $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $19,285 of Social Security disability income as taxable and failed to take Idaho’s $13,784.12 subtraction, then used estimated deductions and unexplained credits. The correct Idaho AGI is $35,280, taxable income is $19,180, and the schedule yields $755.78."
+us,scenario_007,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model double-counted deductions by starting from a federal taxable base and then subtracting another $5,200 Idaho standard deduction. Idaho instead starts from $35,280 after the Social Security subtraction, deducts $16,100 once, and applies the 2026 zero-rate threshold and 5.3% rate to reach $755.78."
+us,scenario_007,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,The model applied an obsolete $120 Idaho Grocery Credit amount instead of the 2026 base amount of $155. Being under age 65 does not reduce this taxpayer’s 2026 credit below $155.
+us,scenario_007,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Idaho has no refundable individual income tax credits. It omitted the refundable Idaho Grocery Credit, which awards this qualifying full-year resident $155."
+us,scenario_007,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,The model used an obsolete $120 Grocery Credit parameter and then inconsistently submitted $140 despite its own derivation. The applicable 2026 base amount for this qualifying resident is $155.
+us,scenario_007,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model misclassified the Idaho Grocery Credit as nonrefundable when claimed on the income tax return. PolicyEngine includes the qualifying taxpayer’s full $155 Grocery Credit in state refundable credits.
+us,scenario_007,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model incorrectly moved the Idaho Grocery Credit out of refundable credits as a nonrefundable or miscellaneous item. The credit is refundable and contributes $155 for this household.
+us,scenario_007,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly conditioned refundability on having no Idaho income tax liability and treated the Grocery Credit as nonrefundable when liability exists. The $155 Grocery Credit remains a refundable credit regardless of whether the taxpayer has enough liability to absorb it, and receipt of SSDI does not reclassify it."
+us,scenario_007,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model misclassified Idaho’s Grocery Credit as nonrefundable in the relevant output. This qualifying full-year resident receives the refundable 2026 amount of $155.
+us,scenario_007,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,The model identified the correct Idaho Grocery Credit pathway but used an obsolete $120 amount. The 2026 base credit is $155.
+us,scenario_007,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly recognized the refundable Idaho Grocery Credit but applied $120 per household member instead of the 2026 amount of $155. With one qualifying member, the result is $155."
+us,scenario_007,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model omitted the Idaho Grocery Credit eligibility pathway. Full-year Idaho residence and the absence of a disqualifying factor produce a refundable credit of $155.
+us,scenario_007,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied an obsolete $120 Idaho Grocery Credit parameter. The qualifying individual’s age and lack of SNAP receipt lead to the full 2026 base amount of $155, not $120."
+us,scenario_007,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model used $120 as the Grocery Credit for a resident under age 65 instead of the applicable 2026 base amount of $155. The taxpayer’s age does not reduce the credit to $120.
+us,scenario_007,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model identified the refundable Grocery Credit but used the wrong annual parameter. Idaho’s 2026 amount for this qualifying resident is $155 rather than $120.
+us,scenario_007,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,The model correctly found Grocery Credit eligibility but applied an obsolete $120 amount. The full 2026 credit is $155.
+us,scenario_007,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model falsely treated the Idaho Grocery Credit as repealed. The program applies in 2026 and awards this qualifying full-year resident a refundable $155 credit.
+us,scenario_007,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to apply Idaho’s refundable Grocery Credit to the full-year resident. The household qualifies for the full $155 base amount.
+us,scenario_007,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model incorrectly required an additional listed qualifying condition and overlooked that Idaho residence throughout the year establishes the relevant Grocery Credit pathway. With no disqualifying factor, the refundable amount is $155."
+us,scenario_007,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model invented an income phaseout that eliminated the Idaho Grocery Credit for this household. The stated pension and SSDI income do not phase out the qualifying resident’s $155 credit.
+us,scenario_007,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model omitted Idaho’s applicable refundable Grocery Credit. This full-year resident qualifies for $155.
+us,scenario_007,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,The model used an obsolete $100 Idaho Grocery Credit amount. The 2026 refundable base amount for this qualifying resident is $155.
+us,scenario_007,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,The model correctly classified the Idaho Grocery Credit as refundable but applied the wrong $100 parameter. The applicable 2026 amount is $155.
+us,scenario_007,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model failed to apply the Idaho Grocery Credit to the qualifying full-year resident. That refundable credit contributes $155.
+us,scenario_007,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for state_refundable_credits. It therefore failed the required output contract rather than completing the Idaho Grocery Credit calculation.
+us,scenario_007,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,The model found the correct Grocery Credit pathway but used an obsolete $120 amount. Idaho’s 2026 credit for this qualifying resident is $155.
+us,scenario_007,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly asserted that Idaho has no state income tax and consequently omitted its refundable credits. Idaho’s Grocery Credit applies to this resident and contributes $155.
+us,scenario_007,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,The model estimated the 2026 Grocery Credit as roughly $140 instead of applying the exact statutory parameter represented in the calculation. The full base amount is $155.
+us,scenario_007,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model overlooked the Idaho Grocery Credit applicable to this household. The qualifying full-year resident receives a refundable $155 credit.
+us,scenario_007,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,The model incorrectly concluded that no Idaho refundable credit applies. The household qualifies throughout the year for the $155 Idaho Grocery Credit.
+us,scenario_008,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only the under-19 age rule and CHIP income ceiling, then treated income below 350% FPL as sufficient. It omitted the prior Medicaid determination: child1 qualifies for Medicaid as an OLDER_CHILD, which precludes CHIP."
+us,scenario_008,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated low household income and age under 19 as independently establishing CHIP eligibility. It failed to test Medicaid first; child1's OLDER_CHILD Medicaid eligibility makes the child ineligible for CHIP.
+us,scenario_008,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared income directly with the CHIP ceiling and added age and lack of listed coverage as sufficient conditions. It skipped CHIP's requirement that the child not qualify for Medicaid, while child1 qualifies through New Jersey's OLDER_CHILD category."
+us,scenario_008,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a child in a low-income household qualifies for CHIP. The correct eligibility sequence first assigns child1 to Medicaid as an OLDER_CHILD, and that Medicaid eligibility excludes CHIP."
+us,scenario_008,child1_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model concluded that being under 19 and below New Jersey's CHIP income limit was sufficient. It omitted the Medicaid-exclusion step: child1 is Medicaid eligible as an OLDER_CHILD and therefore cannot qualify for CHIP.
+us,scenario_008,child1_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated age 17 plus low income with CHIP eligibility. It failed to apply Medicaid precedence, under which child1 qualifies as an OLDER_CHILD and is consequently excluded from CHIP."
+us,scenario_008,child1_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model treated household income far below the CHIP ceiling as establishing eligibility. Income below that ceiling does not override the Medicaid exclusion: child1 qualifies for Medicaid in the OLDER_CHILD category, so CHIP eligibility is false."
+us,scenario_008,child1_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model relied on age and MAGI being within New Jersey's CHIP limits. It never applied the requirement that CHIP cover only children ineligible for Medicaid; child1 is Medicaid eligible as an OLDER_CHILD, which precludes CHIP."
 us,scenario_008,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separately listed Medicaid eligibility trigger and failed to apply the age-based OLDER_CHILD pathway. Child 1 is age 17 and a tax-unit dependent, and the household's MAGI of 0.55 times FPL satisfies New Jersey's income limit for that category."
 us,scenario_008,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model applied the wrong WIC age rule: child eligibility ends at age five, so Child 1 at age 17 is not eligible. Its statement that age 17 falls within the WIC child age range directly reverses the categorical requirement."
 us,scenario_008,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly classified a 17-year-old as an age-eligible WIC child; WIC child eligibility is limited to children younger than five. Its income conclusion cannot establish eligibility after Child 1 fails this categorical age test.
-us,scenario_008,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated income below New Jersey's CHIP ceiling as sufficient for eligibility. It omitted the prior Medicaid determination: Child 2 qualifies under the OLDER_CHILD Medicaid category and is therefore excluded from CHIP.
-us,scenario_008,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied the NJ FamilyCare upper income limit without first checking whether Child 2 qualified for Medicaid. The OLDER_CHILD Medicaid pathway applies, and existing Medicaid eligibility disqualifies the child from CHIP."
-us,scenario_008,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model concluded that age under 19, low income, and no listed private insurance established CHIP eligibility. It missed the controlling exclusion that Child 2's OLDER_CHILD Medicaid eligibility makes the child ineligible for CHIP."
-us,scenario_008,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model equated being a low-income child with CHIP eligibility and skipped the Medicaid-first eligibility sequence. Child 2 is Medicaid-eligible under OLDER_CHILD, which precludes CHIP eligibility."
-us,scenario_008,child2_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model treated the under-19 age rule and income below the CHIP limit as sufficient conditions. It failed to apply the exclusion for children already eligible for Medicaid; Child 2 qualifies through the OLDER_CHILD category.
-us,scenario_008,child2_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model used age and low household income as a shortcut to CHIP eligibility. It omitted Child 2's OLDER_CHILD Medicaid eligibility, which automatically makes the child ineligible for CHIP."
-us,scenario_008,child2_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model checked household MAGI against New Jersey's CHIP income limits but did not check Medicaid eligibility first. Child 2 qualifies for Medicaid under OLDER_CHILD and consequently cannot qualify for CHIP.
+us,scenario_008,child2_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied New Jersey's CHIP age and 355% FPL ceiling without first testing Medicaid eligibility. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which disqualifies the child from CHIP."
+us,scenario_008,child2_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated income below New Jersey's CHIP ceiling as sufficient for eligibility and omitted CHIP's requirement that the child not qualify for Medicaid. Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore is not CHIP-eligible.
+us,scenario_008,child2_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age, income, and the absence of listed private insurance but failed to apply the Medicaid-exclusion step. Child 2's eligibility for Medicaid under the OLDER_CHILD category automatically makes CHIP eligibility false."
+us,scenario_008,child2_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model equated being a low-income child with CHIP eligibility and omitted the Medicaid-first eligibility pathway. Child 2 qualifies for Medicaid under the OLDER_CHILD category, so CHIP is unavailable."
+us,scenario_008,child2_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated being under 19 and below New Jersey's CHIP income limit as sufficient conditions. It failed to recognize that Child 2 qualifies for Medicaid under the OLDER_CHILD category, which precludes CHIP."
+us,scenario_008,child2_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,The model used age and low household income as a shortcut for CHIP eligibility without evaluating Medicaid eligibility. Child 2 is Medicaid-eligible under the OLDER_CHILD category and consequently cannot qualify for CHIP.
+us,scenario_008,child2_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model compared income only with New Jersey's CHIP ceiling and ignored that CHIP covers children who do not qualify for Medicaid. At this low income, Child 2 qualifies for Medicaid under the OLDER_CHILD category, making CHIP eligibility false."
+us,scenario_008,child2_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated MAGI within New Jersey's CHIP limits as sufficient and skipped the prior Medicaid determination. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which disqualifies the child from CHIP."
 us,scenario_008,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separately stated Medicaid trigger instead of applying New Jersey's mandatory OLDER_CHILD pathway. Child 2 is age 14, is a tax-unit dependent, and has household MAGI of 0.55 times FPL, which satisfies that pathway's income test."
 us,scenario_008,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a WIC child age range of 1–18 and therefore treated a 14-year-old as categorically eligible. WIC child eligibility ends at age five, and housing-assistance receipt does not override that age restriction."
 us,scenario_008,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 14 as within the WIC child age range. WIC covers infants and children under five, so Child 2 is categorically ineligible."
 us,scenario_008,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated a 14-year-old as age-eligible for WIC and asserted that household income met the limit. Child 2 fails WIC's under-five age requirement, and the benchmark income computation also places the household above the applicable income threshold."
-us,scenario_008,child3_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated low income and age as sufficient for CHIP and failed to test Medicaid eligibility first. Child3 qualifies for Medicaid under the OLDER_CHILD category, which precludes CHIP eligibility; housing-assistance receipt does not establish CHIP eligibility."
-us,scenario_008,child3_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only NJ FamilyCare's CHIP upper-income ceiling and concluded that income below 355% FPL establishes CHIP eligibility. It omitted the Medicaid screen: child3 qualifies under the OLDER_CHILD Medicaid category, and that eligibility precludes CHIP."
-us,scenario_008,child3_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model equated being below New Jersey's CHIP upper-income limit with CHIP eligibility. It failed to apply the mutually exclusive Medicaid pathway, under which child3 is eligible as an OLDER_CHILD and therefore ineligible for CHIP."
-us,scenario_008,child3_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model checked age and the 355% FPL CHIP ceiling but omitted the requirement that a CHIP child not qualify for Medicaid. At this household income, child3 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false."
-us,scenario_008,child3_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated age, lack of listed coverage, and income below CHIP's ceiling as a complete eligibility test. It failed to recognize that child3's low income establishes OLDER_CHILD Medicaid eligibility, which bars CHIP regardless of unlisted health coverage."
-us,scenario_008,child3_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child is CHIP-eligible without distinguishing Medicaid from CHIP. The correct categorical sequence places child3 in the OLDER_CHILD Medicaid group, and Medicaid eligibility makes CHIP eligibility false."
-us,scenario_008,child3_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from child3's age and low household income. It omitted New Jersey's Medicaid determination: child3 qualifies under the OLDER_CHILD category, which excludes the child from CHIP."
-us,scenario_008,child3_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model applied CHIP's 355% FPL ceiling without first screening child3 for Medicaid, even though the much lower income establishes OLDER_CHILD Medicaid eligibility and therefore bars CHIP. It also incorrectly added an assumed American Opportunity Credit to MAGI; a tax credit is not income, and the prompt supplied no tuition or scholarship amount."
-us,scenario_008,child3_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model treated household MAGI within the CHIP income limit as dispositive. It failed to apply the Medicaid-CHIP exclusivity rule: child3 qualifies for Medicaid as an OLDER_CHILD and consequently is not CHIP-eligible.
+us,scenario_008,child3_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated low income, age, and housing-assistance receipt as sufficient for CHIP without checking Medicaid first. Child 3 qualifies for Medicaid under New Jersey's OLDER_CHILD category, which makes the child ineligible for CHIP."
+us,scenario_008,child3_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied only NJ FamilyCare's approximately 355% FPL upper limit and concluded that every child below it qualifies for CHIP. It omitted the Medicaid-exclusion step: this income places Child 3 in the OLDER_CHILD Medicaid category, so CHIP eligibility is false."
+us,scenario_008,child3_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model conflated the combined NJ FamilyCare coverage ceiling with CHIP eligibility and stopped after finding income below 355% FPL. Child 3 instead qualifies for Medicaid as an OLDER_CHILD, and Medicaid eligibility precludes CHIP."
+us,scenario_008,child3_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model used age under 19 and income below 355% FPL as a complete CHIP test. It failed to determine that Child 3 qualifies for Medicaid under the OLDER_CHILD threshold, which makes CHIP unavailable."
+us,scenario_008,child3_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,The model treated absence of listed health coverage plus income below New Jersey's CHIP ceiling as establishing eligibility. Current coverage is not the controlling exclusion here: Child 3's eligibility for Medicaid in the OLDER_CHILD category precludes CHIP.
+us,scenario_008,child3_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child qualifies for CHIP and omitted the mutually exclusive Medicaid determination. The low household income makes Child 3 Medicaid-eligible as an OLDER_CHILD, so CHIP eligibility is 0."
+us,scenario_008,child3_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from Child 3's age and low household income. Those facts instead establish Medicaid eligibility under the OLDER_CHILD category, and Medicaid eligibility disqualifies the child from CHIP."
+us,scenario_008,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,The model compared income only with New Jersey's approximately 355% FPL CHIP ceiling. It failed to apply the lower-income Medicaid pathway first; at this income Child 3 qualifies as an OLDER_CHILD for Medicaid and therefore cannot qualify for CHIP.
+us,scenario_008,child3_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated income below the 355% FPL ceiling and lack of listed coverage as sufficient for CHIP, omitting the Medicaid exclusion. It also incorrectly added an American Opportunity Credit and invented education expenses to calculate MAGI, but Child 3 is Medicaid-eligible as an OLDER_CHILD even using the stated household income, which precludes CHIP."
+us,scenario_008,child3_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model stopped after finding Child 3 under age 19 and household MAGI within the broad CHIP income limit. It omitted that the same low income qualifies Child 3 for Medicaid under the OLDER_CHILD category, making the child ineligible for CHIP."
 us,scenario_008,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required an explicit Medicaid eligibility trigger instead of applying New Jersey's older-child MAGI pathway. Child 3 is a 12-year-old dependent in the OLDER_CHILD category, and household MAGI of 0.55 times FPL satisfies that category's income test."
 us,scenario_008,child3_wic_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 12 as within WIC's child category. WIC child eligibility ends at age five, so Child 3 fails the categorical age requirement regardless of income."
 us,scenario_008,child3_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied both requirements: a 12-year-old is outside WIC's under-five child category, and household income of approximately $30,915 exceeds the applicable 185% federal-poverty income limit. Its eligibility conclusion therefore fails both the age and income tests."
-us,scenario_008,child4_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated income below New Jersey's CHIP ceiling as sufficient for CHIP eligibility. It failed to apply the prior Medicaid determination: child4 qualifies under the OLDER_CHILD Medicaid category, which makes the child ineligible for CHIP."
-us,scenario_008,child4_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model checked age and the upper CHIP income limit but omitted CHIP's requirement that the child not qualify for Medicaid. Child4's age and approximately $30,915 household income place the child in New Jersey's OLDER_CHILD Medicaid category, so CHIP eligibility is excluded."
-us,scenario_008,child4_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model compared income only with the 355% FPL CHIP ceiling and incorrectly concluded that satisfying age, residency, and that ceiling established eligibility. It skipped the lower-income Medicaid pathway: child4 is Medicaid-eligible as an OLDER_CHILD and therefore cannot be CHIP-eligible."
-us,scenario_008,child4_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being far below New Jersey's CHIP upper income limit as qualification for CHIP. That low income instead places child4 within the OLDER_CHILD Medicaid threshold, and Medicaid eligibility bars CHIP eligibility."
-us,scenario_008,child4_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low income and child status as a shortcut to CHIP eligibility without testing Medicaid eligibility first. Child4 qualifies for Medicaid through New Jersey's OLDER_CHILD category, so the mutually exclusive CHIP rule produces ineligibility."
-us,scenario_008,child4_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from age 10 and low household income. It omitted that the same facts qualify child4 for Medicaid under the OLDER_CHILD category, which excludes the child from CHIP."
-us,scenario_008,child4_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model compared its MAGI estimate with the 355% FPL CHIP ceiling and treated the absence of listed health coverage as sufficient to avoid disqualification. The controlling exclusion is Medicaid eligibility, not current health coverage: child4 qualifies for New Jersey Medicaid as an OLDER_CHILD and is therefore ineligible for CHIP."
-us,scenario_008,child4_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated household MAGI within the CHIP limit as the complete eligibility test. It failed to determine that child4 qualifies for Medicaid under New Jersey's OLDER_CHILD category, which precludes CHIP eligibility."
+us,scenario_008,child4_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated New Jersey's CHIP upper-income limit as a standalone eligibility test. It omitted the Medicaid screen: child4 qualifies under the OLDER_CHILD Medicaid category at this income, which excludes the child from CHIP."
+us,scenario_008,child4_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model applied only CHIP's age and income requirements and stopped after finding income below roughly 350% FPL. It failed to apply the mutually exclusive Medicaid condition: child4 is Medicaid-eligible under the OLDER_CHILD category and therefore not CHIP-eligible.
+us,scenario_008,child4_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly inferred CHIP eligibility from being under 19 and below CHIP's 355% FPL ceiling. The much lower household income places child4 in New Jersey's OLDER_CHILD Medicaid category, and Medicaid eligibility bars CHIP eligibility."
+us,scenario_008,child4_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated income below New Jersey's 355% FPL CHIP ceiling as sufficient to qualify. It skipped the prior Medicaid determination, under which child4 qualifies as an OLDER_CHILD and is consequently excluded from CHIP."
+us,scenario_008,child4_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low household income as direct support for CHIP eligibility. At this income, low income instead establishes child4's Medicaid eligibility under the OLDER_CHILD category, which makes CHIP unavailable."
+us,scenario_008,child4_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model concluded that age 10 and low income qualify child4 for CHIP without screening for Medicaid. Child4 qualifies for Medicaid under New Jersey's OLDER_CHILD category, so the program-exclusivity rule makes the child ineligible for CHIP."
+us,scenario_008,child4_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model compared income only with CHIP's approximately 355% FPL upper limit. Income near 54% FPL places child4 within the OLDER_CHILD Medicaid threshold, and Medicaid eligibility precludes CHIP eligibility."
+us,scenario_008,child4_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated age, income below 355% FPL, and lack of listed health coverage as sufficient for CHIP. It omitted the controlling Medicaid exclusion: child4 qualifies for Medicaid under the OLDER_CHILD category, so absence of existing coverage does not create CHIP eligibility."
+us,scenario_008,child4_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility solely because the child's age and household MAGI fall within CHIP's broad income range. It failed to determine first that child4 is Medicaid-eligible under the OLDER_CHILD category, which makes the child ineligible for CHIP."
 us,scenario_008,child4_head_start_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model asserted that age 10 is eligible for Head Start without applying the program's preschool-age restriction. Child 4 is too old for Head Start, so the eligibility value is 0."
 us,scenario_008,child4_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required an explicit Medicaid-triggering fact instead of applying New Jersey's mandatory OLDER_CHILD pathway. Child4 is age 10, and household MAGI of 0.55 times FPL is below the applicable income threshold, so the eligibility result is Yes."
 us,scenario_008,child4_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC’s child age range. WIC child eligibility ends at age five, so Child 4 is categorically ineligible."
 us,scenario_008,child4_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model invented a WIC eligibility category for school-age children. A 10-year-old does not satisfy WIC’s categorical age requirement, which covers children only until age five."
 us,scenario_008,child4_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model both treated a 10-year-old as categorically eligible for WIC and incorrectly placed the household within the income limit. Child 4 is outside WIC’s under-five child category, and PolicyEngine’s approximately $30,915 household income exceeds its applicable WIC ceiling."
-us,scenario_008,child5_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the CHIP income ceiling directly and incorrectly treated low income and housing assistance as supporting CHIP eligibility. It skipped the prior YOUNG_CHILD Medicaid determination, which makes this child ineligible for CHIP regardless of being below the CHIP ceiling."
-us,scenario_008,child5_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the five-year-old was not in the Medicaid-only age-and-income category and assigned CHIP based on the roughly 355% FPL ceiling. At this income, the child qualifies for Medicaid under YOUNG_CHILD, and that Medicaid eligibility excludes CHIP."
-us,scenario_008,child5_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being under New Jersey's CHIP income limit as sufficient for CHIP eligibility. It omitted the Medicaid screen: the child's age and household income establish YOUNG_CHILD Medicaid eligibility, which disqualifies the child from CHIP."
-us,scenario_008,child5_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model compared income only with the approximately 355% FPL CHIP ceiling and treated lack of other health coverage as dispositive. It failed to apply the lower-income YOUNG_CHILD Medicaid pathway first; Medicaid eligibility itself bars CHIP.
-us,scenario_008,child5_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated NJ FamilyCare's combined Medicaid/CHIP coverage umbrella with eligibility for the distinct CHIP category. The low income places Child 5 in Medicaid's YOUNG_CHILD category, not CHIP, and Medicaid eligibility excludes CHIP."
-us,scenario_008,child5_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low household income as a shortcut to CHIP eligibility. For this five-year-old, that low income instead establishes YOUNG_CHILD Medicaid eligibility, which makes the CHIP result zero."
-us,scenario_008,child5_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model assumed that a low-income New Jersey child qualifies for CHIP without first testing Medicaid. Child 5 qualifies under the YOUNG_CHILD Medicaid category, and CHIP excludes children who are Medicaid-eligible."
-us,scenario_008,child5_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model compared its MAGI estimate with the 355% FPL CHIP ceiling and treated the absence of other coverage as sufficient. It omitted the controlling Medicaid determination: at this age and income, Child 5 is YOUNG_CHILD Medicaid-eligible and therefore CHIP-ineligible."
-us,scenario_008,child5_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated income within New Jersey's CHIP limit as sufficient eligibility. It failed to recognize that the same age-and-income facts first qualify Child 5 for Medicaid under YOUNG_CHILD, which categorically prevents CHIP eligibility."
+us,scenario_008,child5_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the CHIP income ceiling directly and omitted the prerequisite that a child must not qualify for Medicaid. Child 5 qualifies for Medicaid under the YOUNG_CHILD category, so neither the low income nor housing assistance establishes CHIP eligibility."
+us,scenario_008,child5_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Child 5 was not in a Medicaid-only age-and-income category. At age 5 and this income, the child qualifies for Medicaid under YOUNG_CHILD, which automatically precludes CHIP eligibility."
+us,scenario_008,child5_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated income below New Jersey's CHIP ceiling and age under 19 as sufficient conditions. It failed to test Medicaid first: Child 5 qualifies under the YOUNG_CHILD Medicaid category and therefore cannot qualify for CHIP.
+us,scenario_008,child5_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model compared household income only with the 355% FPL CHIP ceiling and treated the absence of other health coverage as decisive. It omitted Medicaid eligibility as an exclusion; Child 5 qualifies for Medicaid through YOUNG_CHILD and is consequently ineligible for CHIP.
+us,scenario_008,child5_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated combined NJ FamilyCare Medicaid/CHIP coverage with eligibility for the benchmark's distinct CHIP category. Child 5's low income establishes Medicaid eligibility under YOUNG_CHILD, and Medicaid eligibility excludes CHIP."
+us,scenario_008,child5_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used low income and child status as a shortcut to CHIP eligibility without checking Medicaid. The correct eligibility sequence places Child 5 in the YOUNG_CHILD Medicaid category, which makes CHIP eligibility false."
+us,scenario_008,child5_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated being a low-income New Jersey child with qualifying for CHIP. Child 5 instead qualifies for Medicaid under YOUNG_CHILD, and CHIP covers children who do not qualify for Medicaid."
+us,scenario_008,child5_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,The model compared the household's FPL percentage only with New Jersey's CHIP upper limit. It omitted the lower-income Medicaid pathway: Child 5 qualifies for Medicaid under YOUNG_CHILD and is therefore excluded from CHIP.
+us,scenario_008,child5_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated MAGI below 355% FPL and no listed health coverage as sufficient for CHIP. It failed to determine that Child 5 qualifies for Medicaid through the YOUNG_CHILD category, which independently disqualifies the child from CHIP."
+us,scenario_008,child5_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model used the CHIP income limit as the sole eligibility test. At age 5 and household income of approximately $30,915, Child 5 qualifies for Medicaid under YOUNG_CHILD, so the required Medicaid-ineligibility condition for CHIP is not met."
 us,scenario_008,child5_early_head_start_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated Early Head Start as serving children from birth through age 5. Because Child 5 is age 5, the child is outside Early Head Start's under-age-3 category; income and housing assistance do not cure that categorical age failure."
 us,scenario_008,child5_head_start_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model incorrectly restricted preschool Head Start to ages 3–4 and treated age five as an automatic exclusion. The benchmark’s Head Start age rule includes this 5-year-old, who also qualifies through SNAP participation and household income below 130% of the federal poverty guideline."
 us,scenario_008,child5_head_start_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model substituted a presumed usual preschool range for the benchmark’s Head Start age rule and incorrectly excluded the child at age five. With age five included and both the SNAP categorical and direct income pathways satisfied, the child is eligible."
@@ -388,50 +409,53 @@ us,scenario_008,child5_wic_eligible,grok-4.3,llm_error,age_disability,False,"The
 us,scenario_008,child5_wic_eligible,inkling,llm_error,age_disability,False,"The model conflated “under 5” with “through age 5.” A child reported as age 5 has reached the cutoff and is outside WIC's child category, irrespective of the income ratio."
 us,scenario_008,child5_wic_eligible,minimax-m3,llm_error,age_disability,False,"The model declared categorical eligibility solely from age 5, but the WIC child category ends at the fifth birthday. Its poverty-threshold calculation cannot overcome failure of that age requirement."
 us,scenario_008,child5_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model treated age 5 as satisfying the WIC categorical requirement and proceeded to the income test. Child 5 is already beyond the under-5 child category, so a qualifying income level would not establish eligibility."
-us,scenario_008,child6_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied NJ FamilyCare's CHIP age and upper-income limit without first testing Medicaid eligibility. Child 6 qualifies for Medicaid under the YOUNG_CHILD category, which precludes CHIP eligibility."
-us,scenario_008,child6_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated income below the CHIP ceiling and age under 19 as sufficient for CHIP. It omitted the prior Medicaid screen: the 1-year-old qualifies under New Jersey's YOUNG_CHILD Medicaid category and therefore cannot qualify for CHIP.
-us,scenario_008,child6_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model compared household income only with New Jersey's CHIP limit for children. It failed to apply the YOUNG_CHILD Medicaid pathway, under which Child 6 is Medicaid-eligible and consequently CHIP-ineligible."
-us,scenario_008,child6_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model listed age, income, and absence of other health coverage as all CHIP conditions but omitted the controlling Medicaid exclusion. Child 6's eligibility for Medicaid under the YOUNG_CHILD category disqualifies the child from CHIP regardless of the absence of existing coverage."
-us,scenario_008,child6_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated the combined NJ FamilyCare program's broad child income ceiling with CHIP eligibility specifically. At age 1 and this income, Child 6 enters NJ FamilyCare through the YOUNG_CHILD Medicaid category, and Medicaid eligibility excludes CHIP."
-us,scenario_008,child6_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child qualifies for CHIP and never separated Medicaid from CHIP. The correct categorical sequence assigns Child 6 to Medicaid as a YOUNG_CHILD, making CHIP eligibility false."
-us,scenario_008,child6_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model stopped after finding income below New Jersey's CHIP ceiling. It omitted the prerequisite that CHIP covers children ineligible for Medicaid; Child 6 qualifies for Medicaid under the YOUNG_CHILD category.
-us,scenario_008,child6_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated young age and low household income with CHIP eligibility. Those facts instead establish Child 6's Medicaid eligibility under the YOUNG_CHILD category, which bars CHIP eligibility."
-us,scenario_008,child6_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model relied on low-income indicators and lack of existing health coverage without evaluating Medicaid eligibility. Child 6 qualifies for Medicaid as a YOUNG_CHILD, and the absence of other coverage does not override CHIP's exclusion of Medicaid-eligible children."
-us,scenario_008,child6_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model treated MAGI within the CHIP limit as dispositive. It skipped the Medicaid determination that comes first: Child 6 qualifies under the YOUNG_CHILD Medicaid category and is therefore ineligible for CHIP.
+us,scenario_008,child6_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model compared household income only with New Jersey's CHIP ceiling and treated falling below that ceiling as sufficient. It failed to apply the prior Medicaid screen: child6 qualifies under the YOUNG_CHILD Medicaid category, which excludes the child from CHIP."
+us,scenario_008,child6_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied CHIP's age and upper-income criteria without testing Medicaid eligibility first. Child6's age and low household income place the child in the YOUNG_CHILD Medicaid category, so CHIP eligibility is false."
+us,scenario_008,child6_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated satisfying CHIP age and income limits as conclusive. It omitted the rule that CHIP covers children who do not qualify for Medicaid; child6 already qualifies for Medicaid under the YOUNG_CHILD category.
+us,scenario_008,child6_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model listed age, income, and lack of other health coverage as all CHIP conditions but omitted Medicaid ineligibility. Child6 is Medicaid-eligible under the YOUNG_CHILD category, which independently bars CHIP eligibility regardless of being uninsured."
+us,scenario_008,child6_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated eligibility for the combined NJ FamilyCare system with eligibility for its CHIP pathway. At this age and income, child6 belongs in the Medicaid YOUNG_CHILD pathway, and Medicaid eligibility precludes CHIP."
+us,scenario_008,child6_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used the shortcut that a low-income child qualifies for CHIP. Low income instead makes this 1-year-old eligible for Medicaid under the YOUNG_CHILD category, and that eligibility excludes the child from CHIP."
+us,scenario_008,child6_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model stopped after finding household income below New Jersey's CHIP limit. It failed to test the mutually exclusive Medicaid pathway, under which child6 qualifies as a YOUNG_CHILD and therefore cannot qualify for CHIP."
+us,scenario_008,child6_chip_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model equated being a low-income child in New Jersey with CHIP eligibility. Child6's low income and age establish Medicaid eligibility under the YOUNG_CHILD category, which makes CHIP eligibility false."
+us,scenario_008,child6_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model treated income far below the CHIP ceiling as satisfying CHIP eligibility. That income places the 1-year-old in Medicaid's YOUNG_CHILD category, and CHIP is unavailable once Medicaid eligibility is established."
+us,scenario_008,child6_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility from housing assistance, low income, age, and lack of other coverage while omitting the Medicaid exclusion. Child6 qualifies for Medicaid under the YOUNG_CHILD category, so the child is not eligible for CHIP."
+us,scenario_008,child6_chip_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model used household MAGI being within the CHIP income limit as the complete test. It skipped the antecedent Medicaid determination: child6 is Medicaid-eligible under the YOUNG_CHILD category and is consequently ineligible for CHIP.
 us,scenario_008,child6_early_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model inverted the Early Head Start age rule: being age 1 establishes that Child 6 is within the under-3 target group rather than excluding the child. It also failed to apply the household-income test, under which approximately $30,915 is below 100% of the federal poverty line for the household."
 us,scenario_008,child6_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 1 as within the preschool-age Head Start range. A one-year-old falls under Early Head Start rather than Head Start, and age alone does not establish income eligibility."
 us,scenario_008,child6_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a separate explicit Medicaid-triggering fact and failed to apply the age-based YOUNG_CHILD pathway. Child 6 is age 1 and a dependent, and the household's MAGI of 0.55 times FPL satisfies the New Jersey income test for that category."
 us,scenario_008,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model incorrectly placed the household inside the EITC phaseout and reduced the credit below $8,231. It also failed to carry its own $4,199 refundable-CTC calculation into the submitted total, which should be $4,202.61 under the engine's earned-income computation."
-us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model reduced the three-or-more-child EITC to $1,395 instead of applying the $8,231 amount and then inconsistently replaced its computed $4,245 CTC with $1,305. Those unsupported reductions omitted $9,733.61 of refundable credits."
-us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model's component reasoning supports roughly $12,469, not its submitted $13,167. It used an incorrect $8,267 EITC parameter instead of $8,231 and failed to submit the sum of the components it actually calculated."
-us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented a $1,000 refundable AOTC despite the explicit zero default for unlisted education expenses, then reduced the refundable CTC from about $4,203 to $388 without a valid limitation. The correct components are $8,231 of EITC, $4,202.61 of refundable CTC, and zero refundable AOTC."
-us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model understated the EITC as roughly $7,700 rather than $8,231 and invented a small refundable AOTC even though qualified education expenses are zero. The refundable CTC is $4,202.61, producing $12,433.61 without any AOTC."
-us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted 2025 EITC parameters and an obsolete $26,511 phaseout threshold for the requested 2026 rules, reducing the EITC to $7,178. Under the 2026 parameters the EITC is $8,231, and the refundable CTC is $4,202.61."
-us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model misread the refundable-CTC phase-in limit: 15% of aggregate earned income above $2,500 limits the household credit to $4,202.61, not $8,500. It also assumed a $1,000 refundable AOTC despite zero qualified expenses and used the wrong EITC amount."
-us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an incorrect $3,000 refundable-CTC threshold and understated the three-or-more-child EITC as $7,787. The applicable results are an $8,231 EITC and $4,202.61 refundable CTC."
-us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly reduced the EITC to $7,691 instead of applying the 2026 three-or-more-child amount of $8,231. Its refundable-CTC estimate was close, but the engine's earned-income calculation yields $4,202.61."
-us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $11,400 total does not equal the applicable maximum EITC plus the earned-income-limited refundable CTC. Those components are $8,231 and $4,202.61, totaling $12,433.61."
-us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model understated the refundable CTC as about $4,124 instead of applying the $2,500 threshold to the proper earned-income base, which yields $4,202.61. Together with the exact $8,231 EITC, the total is $12,433.61."
-us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied the wrong EITC schedule, producing $7,128 instead of $8,231, and understated the refundable CTC as $4,128 instead of $4,202.61. Both component errors drove the low total."
-us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated earned income as wages plus self-employment income minus the half-SE-tax income-tax deduction, which is not the EITC/ACTC earned-income computation. It also used the 2025 $8,046 EITC maximum; the correct components are $8,231 and $4,202.61."
-us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The unexplained $11,940 total understates the sum of the applicable refundable-credit components. The 2026 EITC is $8,231 and the refundable CTC is $4,202.61, totaling $12,433.61."
-us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value for federal_refundable_credits.
-us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the six explicitly listed children and the household's earned income, both of which establish the EITC and refundable-CTC pathways. The household receives $8,231 of EITC and $4,202.61 of refundable CTC."
-us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed no qualifying income structure was specified even though wages, self-employment income, filing household, and six children were explicitly provided. Those facts produce $8,231 of EITC and $4,202.61 of refundable CTC."
-us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used gross self-employment income in earned income, calculating $30,800 and a $4,245 refundable CTC. The statutory self-employment adjustment produces a $4,202.61 refundable CTC; adding the correctly identified $8,231 EITC gives $12,433.61."
-us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used the obsolete $8,046 EITC amount and gross self-employment income to calculate a $4,245 refundable CTC. The 2026 components are $8,231 and $4,202.61."
-us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,231 EITC but used gross self-employment income in the refundable-CTC phase-in, producing $4,245. The adjusted earned-income base yields $4,202.61."
-us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its $12,476 answer implies the same $8,231 EITC plus a $4,245 CTC obtained from gross wages and gross self-employment income. Applying the required self-employment earned-income adjustment yields a $4,202.61 refundable CTC."
-us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated refundable credits as unavailable once nonrefundable credits reduced income tax to zero. EITC and refundable CTC are payable beyond tax liability, yielding $8,231 and $4,202.61 respectively."
-us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used projected or pre-TCJA parameters, including a $3,000 CTC threshold and an incorrect $8,247 EITC maximum. The requested 2026 rules yield an $8,231 EITC and $4,202.61 refundable CTC."
-us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly subjected the household to an EITC phaseout and reduced the credit to $7,901. The applicable EITC is $8,231, while the refundable CTC is $4,202.61."
-us,scenario_008,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $8,271 EITC maximum and treated gross self-employment income as earned income, producing a $4,245 refundable CTC. The correct components are $8,231 and $4,202.61."
-us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for federal_refundable_credits.
-us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,231 EITC but used $30,800 of gross earnings in the refundable-CTC formula, giving $4,245. The adjusted self-employment earned-income base yields $4,202.61."
-us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an incorrect EITC maximum of about $7,152 and an incorrect $28,120 phaseout threshold, reducing the EITC to $6,588. It also used gross self-employment income for the CTC; the correct components are $8,231 and $4,202.61."
-us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model invented a $1,000 refundable AOTC despite zero qualified education expenses and applied estimated, outdated EITC parameters that reduced the credit to $7,144. The correct total contains only the $8,231 EITC and $4,202.61 refundable CTC."
-us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly limited refundable CTC by purported remaining tax-liability room, even though the ACTC is refundable beyond income-tax liability, and reduced it to $1,600. It also understated the EITC; the correct components are $8,231 and $4,202.61."
+us,scenario_008,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model incorrectly placed the household in the EITC phaseout and reduced the $8,231 credit to about $8,173. It also used $3,694 of net self-employment earnings for the refundable CTC rather than the applicable earned-income calculation that produces $4,202.61."
+us,scenario_008,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model reduced the EITC to $1,395 even though the household receives the full $8,231 three-or-more-child credit. It then contradicted its own $4,245 refundable-CTC calculation by substituting $1,305, omitting most of both components."
+us,scenario_008,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"Its reasoning identifies approximately the right refundable CTC and a near-maximum EITC, but the submitted $13,167 does not equal its stated component sum of roughly $12,469. The correct components are $8,231 and $4,202.61, totaling $12,433.61."
+us,scenario_008,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented a $1,000 refundable American Opportunity Credit despite the instruction that unlisted qualified education expenses equal zero. It also reduced the refundable CTC to $388 instead of applying the earned-income phase-in, which yields $4,202.61."
+us,scenario_008,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model understated the EITC as roughly $7,700 instead of applying the full $8,231 plateau amount. It also added a refundable American Opportunity Credit despite zero listed qualified education expenses and used gross self-employment income in its refundable-CTC phase-in."
+us,scenario_008,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used 2025 EITC parameters, especially a $26,511 phaseout threshold, and therefore reduced the EITC to $7,178. Under the benchmark's 2026 parameters the household receives the full $8,231 EITC, alongside $4,202.61 of refundable CTC."
+us,scenario_008,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated the five-child refundable cap as payable in full and assumed a $1,000 refundable American Opportunity Credit without education expenses. The refundable CTC is instead limited by its household-level earned-income phase-in to $4,202.61, and the AOTC is zero."
+us,scenario_008,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an erroneous $3,000 refundable-CTC threshold and reduced the EITC to $7,787. The applicable calculation produces the full $8,231 EITC and $4,202.61 refundable CTC."
+us,scenario_008,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly phased the EITC down to $7,691 rather than awarding the full $8,231 plateau credit. It also used $3,694 of self-employment earnings in the refundable-CTC phase-in, yielding $4,199 instead of $4,202.61."
+us,scenario_008,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer gives no component calculation and its $11,400 total omits $1,033.61 from the required $8,231 EITC plus $4,202.61 refundable CTC. Calling both credits “full” does not apply the refundable CTC's earned-income limitation or reproduce either traced component."
+us,scenario_008,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly recognized a maximum EITC but rounded it down from $8,231 and understated the refundable CTC as $4,124. The earned-income phase-in produces $4,202.61, so the two components total $12,433.61."
+us,scenario_008,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly reduced the EITC to $7,128 and understated the refundable CTC as $4,128. The household receives the full $8,231 EITC and $4,202.61 refundable CTC."
+us,scenario_008,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the prior-year $8,046 EITC maximum instead of the 2026 amount of $8,231. It also treated earned income as wages plus self-employment income minus the half-SE-tax deduction, rather than using the refundable-CTC earned-income calculation that yields $4,202.61."
+us,scenario_008,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The unexplained $11,940 total omits $493.61 from the traced $8,231 EITC and $4,202.61 refundable CTC. The answer did not calculate the applicable 2026 EITC maximum or the refundable-CTC earned-income phase-in."
+us,scenario_008,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no numeric output for federal_refundable_credits, so the required result was missing."
+us,scenario_008,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the six explicitly listed children and the household's earned income, incorrectly denying both refundable credits. These facts establish qualifying children for the three-or-more-child EITC and five under-17 children for CTC purposes, producing $8,231 and $4,202.61 respectively."
+us,scenario_008,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed that no qualifying income structure was specified even though wages, self-employment income, filing household, and six children were all given. Applying those facts yields an $8,231 EITC and $4,202.61 refundable CTC."
+us,scenario_008,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used the head's full $4,000 self-employment income as earned income, producing a $4,245 refundable CTC. The applicable self-employment earned-income calculation produces $4,202.61; adding the correctly identified $8,231 EITC gives $12,433.61."
+us,scenario_008,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an $8,046 EITC amount instead of the 2026 maximum of $8,231 and treated all $4,000 of self-employment income as earned income for the CTC phase-in. The refundable CTC is $4,202.61, not $4,245."
+us,scenario_008,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly used the $8,231 EITC but included the full $4,000 of self-employment income in CTC earned income, yielding $4,245. The applicable earned-income calculation limits refundable CTC to $4,202.61."
+us,scenario_008,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Its total matches an $8,231 EITC plus a $4,245 refundable CTC, so it used gross self-employment income in the CTC phase-in. The applicable self-employment earned-income calculation yields $4,202.61."
+us,scenario_008,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model treated refundable credits as unavailable once nonrefundable credits reduced income tax to zero. EITC and refundable CTC are payable beyond tax liability, yielding $8,231 and $4,202.61 even when pre-refund income tax is zero."
+us,scenario_008,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an erroneous $3,000 refundable-CTC threshold and an incorrect projected EITC maximum of $8,247. The benchmark applies a $2,500 CTC threshold through the relevant earned-income calculation, producing $4,202.61, and a 2026 EITC of $8,231."
+us,scenario_008,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used an erroneous $3,000 refundable-CTC threshold and rounded the EITC maximum up to $8,250. The correct components are the $8,231 EITC and $4,202.61 refundable CTC."
+us,scenario_008,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly subjected the household to an EITC phaseout and reduced that credit to $7,901. The household receives the full $8,231 EITC, and the applicable CTC earned-income computation produces $4,202.61 rather than $4,199."
+us,scenario_008,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an incorrect projected EITC maximum of $8,271 and included the full $4,000 of self-employment income in the refundable-CTC phase-in. The correct components are $8,231 and $4,202.61."
+us,scenario_008,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output for federal_refundable_credits, so the required result was missing."
+us,scenario_008,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,231 EITC but used gross self-employment income to calculate a $4,245 refundable CTC. The applicable earned-income calculation yields $4,202.61."
+us,scenario_008,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an incorrect EITC maximum of $7,152 and then imposed an inapplicable phaseout, reducing the credit to $6,588. It also used gross self-employment income in calculating $4,245 of refundable CTC instead of the traced $4,202.61."
+us,scenario_008,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used an incorrect projected EITC maximum of $8,262 and treated the full $4,000 of self-employment income as earned income for refundable CTC. The correct amounts are $8,231 and $4,202.61."
+us,scenario_008,federal_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded a $1,000 refundable American Opportunity Credit despite zero listed qualified education expenses and incorrectly phased the EITC down to $7,144. No refundable AOTC applies, and the household instead receives the full $8,231 EITC plus $4,202.61 refundable CTC."
+us,scenario_008,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly limited refundable CTC by residual tax-liability “room”; refundable CTC is payable beyond income-tax liability and equals $4,202.61 under the earned-income phase-in. It also understated the EITC as $6,801 instead of applying the full $8,231 amount, and its stated component sum does not equal its submitted total."
 us,scenario_008,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required a separately provided school-meals eligibility indicator and treated housing assistance as the only relevant benefit fact. It failed to compute the income-based free tier at 55% of the federal poverty guideline and also missed categorical eligibility through the household's computed SNAP eligibility.
 us,scenario_008,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly concluded that income exceeded New Jersey’s Medicaid limit. The head’s MAGI income level is 0.55 FPL, which is below the ACA adult expansion threshold and yields eligibility."
 us,scenario_008,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an explicit eligibility indicator instead of deriving eligibility from the supplied annual household facts. The 42-year-old head falls within New Jersey’s ACA adult expansion category, and the resulting MAGI level of 0.55 FPL qualifies."
@@ -439,71 +463,75 @@ us,scenario_008,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibilit
 us,scenario_008,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly identified that the head is under the ACA expansion income threshold, then invented a housing-assistance disqualification. Housing assistance does not bar Medicaid eligibility, so the head remains eligible through New Jersey’s MAGI-based adult expansion category."
 us,scenario_008,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model explicitly transferred the infant's categorical eligibility to the Head through household membership. WIC requires the applicant personally to be a pregnant, postpartum, or breastfeeding woman, infant, or child under age 5; the 42-year-old Head meets none of those categories and is therefore ineligible despite household income below the limit."
 us,scenario_008,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model substituted NJ unemployment and workforce-development assessments for the applicable employee contributions and used the wrong family-leave rate while assigning zero to temporary disability insurance. The NJ portion is $50.92 of TDI plus $61.64 of FLI, or $112.56, rather than $128.65."
-us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model performed an internally inconsistent NJ calculation, alternately treating 0.765% as $205.02, 7.65% as $2,050.20, and finally reducing the state contribution to an unsupported $2.12. It omitted the actual $50.92 TDI and $61.64 FLI contributions."
-us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,The model stopped at federal FICA because it was uncertain about New Jersey rates. It omitted $50.92 of employee TDI and $61.64 of employee FLI.
-us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly treated all New Jersey mandatory employee payroll taxes as zero. The output includes $50.92 of NJ TDI and $61.64 of NJ FLI in addition to $2,050.20 of federal FICA."
-us,scenario_008,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model excluded NJ employee TDI and FLI from the payroll-tax measure even though the prompt expressly includes mandatory state payroll taxes. It also rounded $26,800 × 7.65% from $2,050.20 to $2,050."
-us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model dismissed NJ TDI and FLI as minimal or outside a traditional payroll-tax definition despite the benchmark's explicit inclusion of mandatory employee state payroll taxes. Those contributions are $50.92 and $61.64, respectively."
-us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,The model identified that NJ has mandatory employee payroll contributions but then entered zero because it did not resolve their rates. The applicable state components are $50.92 of TDI and $61.64 of FLI.
-us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied incorrect NJ rates of 0.10% for TDI and 0.09% for FLI. On $26,800, the applicable contributions are $50.92 and $61.64, totaling $112.56 rather than $50.92."
-us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model used an approximate combined NJ SUI/FLI amount of $138.02 instead of computing the applicable TDI and FLI components. Those components total $112.56, so federal FICA plus NJ payroll tax equals $2,162.76."
-us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model calculated only the 7.65% federal FICA charge and omitted mandatory NJ employee payroll contributions. It also rounded federal FICA down from $2,050.20; NJ TDI and FLI add $112.56."
-us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model estimated a combined NJ SUI/FLI/TDI charge of roughly $138 rather than applying the traced state components. NJ TDI is $50.92 and FLI is $61.64, totaling $112.56."
-us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model estimated NJ UI and FLI at $123.80 instead of applying the relevant TDI and FLI rates. The state contribution is $112.56, composed of $50.92 TDI and $61.64 FLI."
-us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model assigned $138.02 to NJ SUI and FLI, overstating the state portion and using the wrong component set. The applicable NJ employee taxes are $50.92 of TDI and $61.64 of FLI."
-us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,The model included NJ SUI and workforce assessments and produced a $139.32 state amount instead of the applicable TDI and FLI contributions. The correct NJ portion is $112.56.
+us,scenario_008,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model substituted New Jersey unemployment, workforce-development, and approximate family-leave rates for the applicable 2026 state components. The required state amount is $50.92 of temporary disability insurance plus $61.64 of family leave insurance, not its $128.65 estimate."
+us,scenario_008,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model's contradictory rate calculations ended by assigning only a $2.12 New Jersey assessment. New Jersey TDI and FLI on these wages total $112.56, which must be added to $2,050.20 of federal FICA."
+us,scenario_008,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model stopped at federal FICA and omitted New Jersey employee TDI and FLI because it treated their 2026 rates as unavailable. Those mandatory contributions are $50.92 and $61.64, respectively."
+us,scenario_008,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model explicitly set New Jersey employee payroll taxes to zero. It needed to add $112.56 of employee TDI and FLI to the correctly calculated $2,050.20 of federal FICA."
+us,scenario_008,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model excluded New Jersey employee TDI and FLI from the requested measure and also rounded federal FICA down from $2,050.20 to $2,050. The state contributions add $112.56."
+us,scenario_008,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly treated New Jersey's mandatory employee contributions as outside the benchmark's payroll-tax definition. The output expressly includes state payroll taxes, so $50.92 of TDI and $61.64 of FLI must accompany federal FICA."
+us,scenario_008,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model recognized New Jersey employee payroll taxes but then set them to zero because it lacked confidence in its estimated combined rate. The applicable TDI and FLI contributions total $112.56.
+us,scenario_008,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model applied incorrect New Jersey rates, producing only $26.80 of TDI and $24.12 of FLI. The correct components are $50.92 and $61.64."
+us,scenario_008,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used an approximate 0.515% New Jersey SUI/FLI charge of $138.02. The applicable state components are TDI of $50.92 and FLI of $61.64, totaling $112.56."
+us,scenario_008,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model calculated only the 7.65% federal FICA charge and omitted mandatory New Jersey employee payroll taxes. TDI and FLI add $112.56 to the $2,050.20 federal amount."
+us,scenario_008,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model estimated New Jersey SUI, FLI, and TDI at roughly $138 rather than applying the two components used by the output. TDI of $50.92 and FLI of $61.64 total $112.56."
+us,scenario_008,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model estimated New Jersey UI and FLI at $123.80. The applicable state payroll taxes are $50.92 of TDI and $61.64 of FLI, totaling $112.56."
+us,scenario_008,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,The model added $138.02 of New Jersey SUI and FLI instead of the applicable $112.56 of TDI and FLI. Its federal Social Security and Medicare calculations were correct.
+us,scenario_008,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"The model included FLI, SUI, and workforce contributions under an incorrect combined New Jersey amount of $139.32. The output includes $50.92 of TDI and $61.64 of FLI, totaling $112.56."
 us,scenario_008,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, so the required output was missing."
-us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly stated federal components of $1,661.60 and $388.60 and no state contribution, but then submitted $4,097, which does not follow from its own arithmetic. It also omitted the $112.56 NJ TDI and FLI total."
-us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $1,525 is inconsistent with the standard 7.65% employee FICA rate the explanation invokes: federal FICA alone is $2,050.20. The model also omitted $112.56 of NJ TDI and FLI."
-us,scenario_008,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model applied a 0.9425% NJ bundle containing UI and workforce assessments, producing $252.59. The applicable state payroll taxes in this computation are only $50.92 of TDI and $61.64 of FLI, totaling $112.56."
-us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model estimated NJ employee contributions at $253.26, overstating the traced state liability. NJ TDI and FLI are $50.92 and $61.64, for a combined $112.56."
-us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model estimated mandatory NJ contributions at $263.98 instead of applying the TDI and FLI amounts. The state portion is $112.56, so adding it to $2,050.20 of federal FICA yields $2,162.76."
-us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,The model overstated NJ employee payroll contributions as $202.34. The applicable TDI and FLI contributions total $112.56.
-us,scenario_008,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model included only federal FICA and omitted mandatory NJ employee TDI and FLI. It also rounded $2,050.20 to $2,050; the state components add $112.56."
-us,scenario_008,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model used a combined 0.18% NJ TDI/FLI rate, generating only $48.24. The actual components are $50.92 of TDI and $61.64 of FLI, totaling $112.56."
-us,scenario_008,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model asserted that no state payroll taxes applied and calculated only federal FICA. It omitted $50.92 of NJ TDI and $61.64 of NJ FLI and rounded federal FICA down by $0.20.
-us,scenario_008,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model substituted approximate NJ UI and FLI charges totaling about $127 for the applicable TDI and FLI computation. The state portion is $112.56, and the federal components should remain unrounded at $1,661.60 and $388.60."
+us,scenario_008,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model correctly stated federal FICA components totaling $2,050.20 but submitted $4,097, an arithmetic and answer-transcription failure unsupported by its own reasoning. It also omitted $112.56 of New Jersey TDI and FLI."
+us,scenario_008,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $1,525 estimate does not apply the stated 6.2% Social Security and 1.45% Medicare rates to $26,800. Those federal components total $2,050.20, and New Jersey TDI and FLI add $112.56."
+us,scenario_008,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"The model applied a 0.9425% aggregate New Jersey rate and included UI and workforce charges, overstating the state portion as $252.59. The applicable state portion consists only of $50.92 of TDI and $61.64 of FLI."
+us,scenario_008,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,The model overstated New Jersey employee contributions as $253.26. The applicable TDI and FLI amounts total $112.56.
+us,scenario_008,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"The model used an estimated New Jersey contribution of $263.98 instead of calculating the applicable TDI and FLI components. Those components are $50.92 and $61.64, totaling $112.56."
+us,scenario_008,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model overstated mandatory New Jersey employee payroll contributions as $202.34. The correct state addition is $112.56 from TDI and FLI.
+us,scenario_008,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model stopped at federal FICA and omitted New Jersey TDI and FLI. It also rounded $26,800 times 7.65% from $2,050.20 to $2,050."
+us,scenario_008,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model used a combined 0.18% New Jersey TDI/FLI rate, yielding only $48.24. The applicable contributions are $50.92 of TDI and $61.64 of FLI, totaling $112.56."
+us,scenario_008,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"The model estimated New Jersey UI, TDI, and FLI at about $178 rather than applying the output's TDI and FLI components. The correct state total is $112.56, and federal FICA is $2,050.20 rather than the rounded $2,050."
+us,scenario_008,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model incorrectly asserted that no state payroll taxes applied and stopped at rounded federal FICA. New Jersey employee TDI and FLI add $112.56 to the exact $2,050.20 federal amount."
+us,scenario_008,payroll_tax,inkling,llm_error,state_local_rule,False,"The model substituted approximate New Jersey UI and FLI charges totaling about $127 and rounded the federal components. The applicable state taxes are TDI of $50.92 and FLI of $61.64, while exact federal FICA is $2,050.20."
 us,scenario_008,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, so the required output was missing."
-us,scenario_008,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model included NJ UI and workforce assessments and calculated state contributions of $257.95. The applicable state components are $50.92 of TDI and $61.64 of FLI, totaling $112.56."
-us,scenario_008,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model calculated only federal Social Security and Medicare tax and rounded their $2,050.20 total to $2,050. It omitted $112.56 of mandatory NJ TDI and FLI."
-us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model included a $102.51 NJ UI/workforce contribution while setting NJ disability and family-leave contributions to zero based on older parameters. The applicable 2026 state amounts are $50.92 of TDI and $61.64 of FLI, totaling $112.56."
-us,scenario_008,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model correctly calculated federal FICA but stopped there. It omitted the mandatory NJ employee contributions of $50.92 for TDI and $61.64 for FLI.
+us,scenario_008,payroll_tax,kimi-k3,llm_error,state_local_rule,False,"The model included UI and workforce charges and overstated New Jersey employee contributions as $257.95. The applicable state amount is $112.56, consisting of TDI and FLI."
+us,scenario_008,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model calculated federal FICA but omitted mandatory New Jersey employee TDI and FLI. Those state components add $112.56, and the exact federal amount is $2,050.20 rather than $2,050."
+us,scenario_008,payroll_tax,ox-alpha,llm_error,state_local_rule,False,The model explicitly modeled no mandatory New Jersey employee payroll tax. It needed to add $50.92 of TDI and $61.64 of FLI to federal FICA.
+us,scenario_008,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model carried forward a 2024-style UI/workforce rate and included only $102.51 of New Jersey tax while setting DI and FLI to zero. For this 2026 output, TDI is $50.92 and FLI is $61.64, totaling $112.56."
+us,scenario_008,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model stopped after federal Social Security and Medicare taxes and omitted mandatory New Jersey employee contributions. TDI and FLI add $112.56 to its correctly calculated $2,050.20 federal FICA."
 us,scenario_008,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling and housing assistance as sufficient for reduced-price support, without applying the mutually exclusive free-meals tier. At 55% of the federal poverty guideline and with categorical eligibility through SNAP/TANF, the household qualifies for free meals, which makes reduced-price eligibility false."
 us,scenario_008,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract."
 us,scenario_008,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model returned $306, exactly 7.65% of $4,000, despite stating that it applied 15.3%. It effectively used only half the combined Social Security and Medicare self-employment tax rate and also failed to apply the 92.35% statutory earnings factor, instead of calculating $4,000 × 0.9235 × 0.153 = $565.18."
 us,scenario_008,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output, so its response failed the required output contract."
 us,scenario_008,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated gross self-employment tax of $565.18, then incorrectly subtracted a supposed deduction from that tax liability. The deductible half of self-employment tax reduces adjusted gross income for income-tax purposes; it is not a credit against self-employment tax, so the liability remains $565.18."
-us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model's own calculation produced $1,214 per month before it abandoned that result and asserted $687 without a supporting computation. It also used an incorrect $1,756 maximum allotment and omitted the calendar-year change in allotment parameters."
-us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used $1,164 as the maximum monthly allotment for eight people, far below the applicable $1,789–$1,829 amounts. It also omitted the 20% earned-income deduction, leaving countable income overstated."
-us,scenario_008,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an approximate $1,756 monthly maximum instead of the applicable $1,789–$1,829 calendar-year amounts and used the wrong standard deduction, producing net income near $2,040 rather than $1,763.92. Those parameter errors understated the allotment."
-us,scenario_008,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model computed approximately $13,728 from its stated inputs and then reduced the answer to $11,352 through an unsupported adjustment. It also used an incorrect maximum allotment and standard deduction instead of the traced $1,789–$1,829 maximum and $299 monthly standard deduction."
-us,scenario_008,snap,claude-opus-5,llm_error,other,False,"The model's arithmetic is internally inconsistent: a $1,802 maximum less 30% of $1,700 net income equals about $1,292 per month, not $1,040. Its unsupported monthly result therefore understated the annual benefit."
-us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $1,536 as the maximum monthly allotment for eight people instead of the applicable $1,789–$1,829 amounts. It also used a $258 standard deduction rather than $299, so both the maximum benefit and net-income calculation were wrong."
-us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied an unsubstantiated shelter-and-utility deduction that drove annual net income down to roughly $8,800. The traced allowable deductions total $812.33 per month and yield $1,763.92 of monthly net income; unlisted utilities cannot be inferred."
-us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an unsupported $1,900 maximum monthly allotment, overstating the applicable $1,789–$1,829 amounts. Although its net-income figure was close, the inflated maximum produced the excessive benefit."
-us,scenario_008,snap,gemini-3-flash-preview,llm_error,period_annualization,False,"The model rounded the result to $15,300 without applying the two maximum-allotment parameter periods that occur during calendar year 2026. Combining the monthly results under the $1,789–$1,829 maxima yields $15,246.91."
-us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies a monthly benefit near $1,366.67, which exceeds the result obtained from the applicable $1,789–$1,829 maxima after subtracting 30% of $1,763.92. The model supplied no deduction or maximum-allotment calculation supporting its higher figure."
-us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model asserted roughly $1,307 per month without identifying the maximum allotment or calculating the 30% contribution from $1,763.92 of net income. Applying the actual calendar-year allotment parameters produces $15,246.91, not $15,684."
-us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model deducted an unlisted New Jersey utility allowance when the prompt requires every unlisted expense to be zero. The trace allows $812.33 of monthly deductions and leaves $1,763.92 in net income, so its additional shelter deduction overstated SNAP."
-us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $287 standard deduction rather than $299 and a fixed $1,756 maximum allotment rather than the $1,789–$1,829 amounts applicable across 2026. Those parameter errors understated the annual benefit."
-us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer implies a monthly allotment of $1,404 without showing the maximum-allotment or expected-contribution computation. With $1,763.92 of net income and the applicable eight-person maxima, the annual result is $15,246.91."
-us,scenario_008,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated $198 as the entire annual standard deduction rather than applying the $299 deduction each month. It also used a fixed $1,748 maximum allotment, so both net income and the maximum benefit were understated."
-us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model denied a positive benefit without applying New Jersey categorical eligibility or the stated income tests. The household passes both tests, and the maximum allotment less its 30% contribution produces $15,246.91."
-us,scenario_008,snap,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated SNAP deductions and eligibility facts as missing even though the prompt supplies the income, household size, disability status, rent, and zero-default rule needed for the calculation. Applying the earned-income and standard deductions yields $1,763.92 of monthly net income and a positive allotment."
-us,scenario_008,snap,gpt-5.5,llm_error,period_annualization,False,"The model rounded the monthly benefit to $1,260 and multiplied it uniformly across the year. Calendar year 2026 spans maximum-allotment periods of approximately $1,789 and $1,829, so the monthly benefit changes and sums to $15,246.91."
-us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The asserted $1,139 monthly benefit is inconsistent with the applicable eight-person maximum allotment less 30% of $1,763.92. The model understated the maximum-allotment component or overstated countable income without showing either calculation."
-us,scenario_008,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model used an approximate annual allotment without accounting precisely for the maximum-allotment change during calendar year 2026. Applying the traced monthly net income to both parameter periods yields $15,246.91."
-us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model cited the earned-income and standard deductions but did not show their amounts or the applicable maximum allotments. Its $15,408.20 result does not follow from $1,763.92 of monthly net income and the $1,789–$1,829 maxima."
-us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated housing assistance and income as eliminating SNAP eligibility. Housing assistance supports categorical eligibility, and the household passes the gross and net income tests, producing a positive benefit."
-us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"The model projected an annual maximum allotment of roughly $21,870 instead of applying the actual monthly maxima of about $1,789–$1,829 across calendar year 2026. It also used approximate net income rather than the traced $1,763.92 monthly amount."
-us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"The model imposed the ordinary $4,250 SNAP resource limit on the $69,500 bank balance. New Jersey broad-based categorical eligibility removes that asset test for this household, so the assets do not bar SNAP."
-us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"The model used an unsupported $1,806 monthly maximum and approximate $1,765 net income. The trace uses $1,763.92 of net income and maximum allotments that change from about $1,789 to $1,829 during 2026, yielding $15,246.91."
-us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, so the required output was missing."
-us,scenario_008,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly reached $1,763.92 of monthly net income but applied the $1,789 maximum allotment for all twelve months. Calendar year 2026 also includes months with the approximately $1,829 maximum, raising the annual total to $15,246.91."
-us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"The model denied eligibility by applying a $3,750 resource ceiling to the household's bank assets. New Jersey categorical eligibility eliminates the SNAP asset test here, so the $69,500 balance does not disqualify the household."
-us,scenario_008,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $202 monthly standard deduction instead of $299 and then replaced its own computed $15,391 result with an unsupported $14,784 adjustment. Housing assistance does not supply a basis for that downward adjustment, and the applicable allotment parameters produce $15,246.91."
-us,scenario_008,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model used the ordinary 130% FPL gross-income screen and also misstated that threshold for an eight-person household. New Jersey categorical eligibility applies, and the household's $2,576.25 monthly gross income is below the traced $4,512.50 threshold."
+us,scenario_008,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model first derived $1,214 per month, then replaced it without computation with $687 per month. It also used a $1,756 maximum allotment instead of the applicable $1,789–$1,829 amounts and failed to use the traced $1,763.92 net income."
+us,scenario_008,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a maximum allotment of only $1,164 per month for an eight-person household and omitted the 20% earned-income deduction. The applicable maximum is $1,789–$1,829, and allowable deductions reduce monthly net income to $1,763.92."
+us,scenario_008,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model applied an incorrect standard deduction and left net income near $2,040 per month rather than $1,763.92. It also understated the eight-person maximum allotment at $1,756 instead of applying the $1,789–$1,829 calendar-year amounts."
+us,scenario_008,snap,claude-opus-4.8,llm_error,other,False,"The model's own stated formula produced $13,728 annually, but it then imposed an unexplained adjustment to $11,352. It also used a $1,756 maximum allotment and failed to reach the traced $1,763.92 monthly net income."
+us,scenario_008,snap,claude-opus-5,llm_error,other,False,"The model stated net income of roughly $1,700 and a maximum near $1,802, which would produce substantially more than its claimed $1,040 monthly benefit. Its submitted $12,500 is inconsistent with its own formula and omits the calendar-year allotment calculation yielding $15,246.91."
+us,scenario_008,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an eight-person maximum allotment of $1,536 per month, far below the applicable $1,789–$1,829 amounts. It also used a $258 standard deduction instead of the deductions producing $1,763.92 in monthly net income."
+us,scenario_008,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented a large shelter deduction and reduced annual net income to about $8,800 even though rent is below half of adjusted income and produces no excess-shelter deduction. The correct deduction sequence leaves monthly net income at $1,763.92."
+us,scenario_008,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model calculated net income close to the traced amount but used an unsupported $1,900 monthly maximum allotment. Applying the $1,789–$1,829 maximum amounts instead yields the lower annual benefit."
+us,scenario_008,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model gave only a rounded estimate and did not apply the two calendar-year maximum allotment levels and monthly contribution precisely. The traced calculation totals $15,246.91 rather than the rounded $15,300."
+us,scenario_008,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an excessive monthly allotment after the household contribution and provides no deduction or maximum-allotment calculation. Monthly net income of $1,763.92 and maximum allotments of $1,789–$1,829 produce $15,246.91, not $16,400."
+us,scenario_008,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model treated roughly $1,307 as the year-round monthly benefit instead of applying the actual monthly maximum allotments and approximately $528 contribution. The resulting benefits range from about $1,260 to $1,302 and total $15,246.91."
+us,scenario_008,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied a New Jersey utility allowance despite the instruction that unlisted expenses are zero. That unsupported shelter deduction reduced net income too far and inflated SNAP above the amount based on $1,763.92 monthly net income."
+us,scenario_008,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model understated the standard deduction and used a single $1,756 maximum allotment for all twelve months. The traced deductions produce $1,763.92 in net income, and the applicable maximum allotments are $1,789–$1,829."
+us,scenario_008,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted amount implies a monthly benefit of $1,404, above the result of subtracting the approximately $528 contribution from the applicable maximum allotments. The correct monthly range is about $1,260–$1,302."
+us,scenario_008,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated a monthly standard deduction of $198 as though it were an annual deduction, subtracting only $198 from annual income. It therefore overstated net income, and it also understated the maximum allotment at $1,748 per month."
+us,scenario_008,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the income facts could not support a positive benefit. The household passes both income tests, is categorically eligible through TANF non-cash assistance, and receives a positive allotment after the approximately $528 monthly contribution."
+us,scenario_008,snap,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated standard SNAP deductions and eligibility inputs as absent facts that had to be supplied by the prompt. Those deductions are policy parameters: they reduce monthly income to $1,763.92 and yield a positive benefit."
+us,scenario_008,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model rounded the monthly benefit to $1,260 and applied that amount uniformly across the year. PolicyEngine applies maximum allotments that change from $1,789 to $1,829, producing monthly benefits up to about $1,302 and an annual total of $15,246.91."
+us,scenario_008,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $1,139 monthly estimate understates the benefit produced by the applicable maximum allotments. Subtracting approximately $528 from $1,789–$1,829 yields about $1,260–$1,302 per month."
+us,scenario_008,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used a rounded annual estimate without accounting precisely for the maximum-allotment change during calendar year 2026. Applying the monthly $1,789–$1,829 maxima and the approximately $528 contribution totals $15,246.91."
+us,scenario_008,snap,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model overstated the result from the earned-income and standard deductions by using an excessive effective maximum allotment or insufficient household contribution. The traced net income generates an approximately $528 contribution and a $15,246.91 annual benefit."
+us,scenario_008,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated housing assistance and income as eliminating SNAP. The household passes the gross and net income tests and categorical eligibility, while subsidized housing does not itself make the allotment zero."
+us,scenario_008,snap,grok-4.5,llm_error,thresholds_rates,False,"The model projected an annual maximum of $21,870 instead of applying the traced $1,789–$1,829 monthly maxima. It also treated the listed OTC spending as an excess-medical deduction even though the stated expenses do not exceed the deduction floor."
+us,scenario_008,snap,grok-4.6,llm_error,thresholds_rates,False,"The model used a single estimated maximum allotment of $1,759, below both applicable monthly amounts. The correct $1,789–$1,829 maxima, combined with $1,763.92 net income, produce $15,246.91."
+us,scenario_008,snap,grok-build-0.1,llm_error,asset_resource,False,"The model applied the ordinary SNAP resource limit to the $69,500 bank balance. New Jersey categorical eligibility through TANF non-cash assistance removes that asset test for this household."
+us,scenario_008,snap,inkling,llm_error,thresholds_rates,False,"The model used a single estimated $1,806 maximum and rounded the contribution and annualization. The calendar-year calculation instead uses maximum allotments of $1,789–$1,829 and totals $15,246.91."
+us,scenario_008,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output could not be parsed."
+us,scenario_008,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached $1,763.92 in monthly net income but held the maximum allotment at $1,789 for all twelve months. It omitted the increase to $1,829 for the later calendar-year months, understating the annual total."
+us,scenario_008,snap,minimax-m3,llm_error,asset_resource,False,"The model disqualified the household under a $3,750 resource limit. New Jersey categorical eligibility through TANF non-cash assistance eliminates the SNAP asset test, so the bank balance does not bar eligibility."
+us,scenario_008,snap,ox-alpha,llm_error,thresholds_rates,False,"The model used one estimated maximum allotment of $1,795 for the entire year. The applicable monthly maxima change from $1,789 to $1,829, and their precise application yields $15,246.91."
+us,scenario_008,snap,qwen-3.7-max,llm_error,other,False,"The model's stated arithmetic produced $15,391, then replaced it with $14,784 based on an unspecified housing-assistance limitation. Housing assistance does not create that downward adjustment, and the correct monthly deduction and allotment schedule totals $15,246.91."
+us,scenario_008,snap,qwen3.8-max,llm_error,thresholds_rates,False,"The model used a gross-income threshold below $2,567 for an eight-person household and therefore declared the unit ineligible. The actual gross threshold is $4,512.50 per month, and gross income of $2,576.25 passes it."
 us,scenario_008,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the ACA adult expansion pathway to the 41-year-old spouse. Her MAGI income level of 0.55 times FPL is below New Jersey's 138% FPL adult threshold, so the listed age, household, and income facts establish eligibility without any additional benchmark condition."
 us,scenario_008,spouse_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly found income below the 138% FPL threshold but invented a Medicaid disqualification based on receipt of housing assistance. Housing assistance does not bar eligibility under New Jersey's MAGI-based ACA adult expansion category, so the spouse remains eligible."
 us,scenario_008,spouse_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model treated disability alone as automatic Medicare eligibility regardless of age. It failed to require a Medicare-qualifying entitlement or condition for an individual under 65; the spouse is 41 and has no such listed status.
@@ -516,90 +544,95 @@ us,scenario_008,spouse_wic_eligible,claude-opus-4.8,llm_error,categorical_eligib
 us,scenario_008,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model inferred breastfeeding or postpartum status and nutritional risk solely from the presence of a one-year-old, despite the prompt requiring unlisted statuses to be false. With neither pregnancy, breastfeeding, nor postpartum status listed, the spouse fails WIC's categorical test regardless of income."
 us,scenario_008,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly used disability as a WIC categorical qualifier. Disability and income eligibility do not establish WIC eligibility for an adult who is not pregnant, breastfeeding, or postpartum."
 us,scenario_008,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model counted blindness and disability as two separate $1,000 exemptions even though New Jersey provides one $1,000 blind-or-disabled exemption here. It then improperly deducted 18% of rent; the correct $12,000 of exemptions leaves $18,915 taxable and $264.81 of tax."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model misstated income as $26,915, invented standard and pension deductions, and treated the federal American Opportunity Credit as reducing New Jersey tax. New Jersey taxable income is $18,915 after the specified exemptions, producing $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted the spouse's blindness and disability as separate exemptions, yielding $13,000 instead of $12,000. It then invented unspecified nonrefundable credits to erase the remaining liability rather than taxing $18,915 at 1.4%."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model misapplied New Jersey's $20,000 joint-filer gross-income filing threshold as an exemption or credit. Gross income is $30,915, taxable income is $18,915, and the resulting tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model assigned $3,000 of regular exemptions to the spouses and thereby reached $13,000 total exemptions. The joint regular exemption is $2,000 and the blind-or-disabled exemption is $1,000, so total exemptions are $12,000 and taxable income is $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model separately counted blindness and disability, overstating exemptions by $1,000, and then fabricated a dependent-based low-income phase-in that reduces tax to zero. The actual $12,000 exemption total leaves $18,915 taxed at 1.4%."
-us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $30,632 income figure and allowed only $9,000 of exemptions. Including wages, the full $4,000 of self-employment income, $115 of interest, and all $12,000 of exemptions yields $18,915 taxable income."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly reached $18,915 of taxable income and approximately $265 of tentative tax, then improperly subtracted a $50 tenant credit. No such offset enters this pre-refundable-credit computation, so the liability remains $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The zero answer treats the household's income and exemptions as eliminating taxable income. The $12,000 of exemptions reduce $30,915 only to $18,915, which produces $264.81 of tax."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model first overstated exemptions and obtained about $251, then improperly subtracted a $50 tenant property-tax credit. The applicable exemptions total $12,000, and no tenant credit reduces the traced liability of $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model allowed $14,000 of exemptions by overstating the joint regular exemption and separately counting blindness and disability. The correct total is $12,000, leaving $18,915 taxable."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model correctly computed $18,915 of taxable income and $264.81 of tentative tax, then improperly subtracted a $50 property-tax credit. That credit does not reduce this output."
-us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model overstated exemptions by $1,000, using $13,000 rather than $12,000. New Jersey supplies one combined $1,000 blind-or-disabled exemption, leaving $18,915 taxable rather than $17,915."
-us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no answer or reasoning for the requested quantity.
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified exemptions and credits wipe out the tax. The listed New Jersey exemptions total only $12,000, leaving $18,915 of taxable income and $264.81 of liability."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified deductions and credits to reduce the liability to zero. The applicable exemptions leave $18,915 taxable, and no traced nonrefundable credit offsets the resulting $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly claimed a $181.70 medical deduction from New Jersey taxable income. Without that deduction, $30,915 less $12,000 of exemptions equals $18,915, and 1.4% yields $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model treated the New Jersey child tax credit as an offset that eliminates tax before refundable credits. That refundable relief is excluded from this output, leaving the $264.81 tax on $18,915."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model reduced the liability using medical deductions and a renter property-tax credit that do not enter the traced calculation. The computation uses $12,000 of exemptions only and taxes the remaining $18,915 at 1.4%."
-us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $250.81 answer implies taxable income of $17,915, reflecting $13,000 of exemptions. The blind and disabled facts generate one combined $1,000 exemption, so total exemptions are $12,000 and tax is $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model invented a New Jersey standard deduction and unspecified credits that reduce tax to zero. New Jersey's applicable $12,000 of exemptions leave $18,915 taxable and $264.81 due."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used the wrong $30,632 income amount and only $9,000 of exemptions. The correct inputs produce $30,915 of New Jersey AGI and $12,000 of exemptions, leaving $18,915 in the 1.4% bracket."
-us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted the $1,000 blind-or-disabled exemption and applied the 1.75% marginal rate to the entire taxable base. With $12,000 of exemptions, taxable income is $18,915 and lies wholly in the 1.4% bracket."
-us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model used an incorrect $10,000 exemption total and improperly deducted 18% of rent. The applicable exemptions total $12,000, with no renter deduction in the traced calculation, leaving $18,915 taxable."
-us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or reasoning for the requested quantity.
-us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model correctly calculated $18,915 of taxable income and $264.81 of tax, then improperly subtracted a $50 renter property-tax credit. The requested pre-refundable liability remains $264.81."
-us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used the wrong $30,632 income figure and omitted the $1,000 blind-or-disabled exemption. Correct New Jersey AGI is $30,915 and total exemptions are $12,000, producing $18,915 taxable income."
-us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model valued dependents at $1,000 rather than $1,500 each and then invented a nonrefundable low-income reduction that eliminates tax. Six dependent exemptions contribute $9,000, total exemptions equal $12,000, and no such reduction offsets the $264.81 liability."
-us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exemptions and deductions reduce New Jersey taxable income to zero. The applicable exemptions total $12,000 against $30,915 of income, leaving $18,915 taxable and $264.81 due."
-us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly phased down the federal EITC from $8,231 to about $6,110 and applied an outdated NJ CTC calculation limited to two children under age six. It also omitted the $50 renter Property Tax Credit; the correct components are $3,292.40, $2,000, and $50."
-us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely asserted that New Jersey has no refundable state credits. This household receives the refundable NJ EITC, NJ Child Tax Credit, and renter Property Tax Credit totaling $5,342.40."
-us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The submitted $9,492 does not follow from the model's own estimated $2,880 EITC and $1,000 CTC, and it omitted the $50 renter Property Tax Credit. It also restricted the NJ CTC to one child under six instead of applying $2,000 across four qualifying dependent children."
-us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model used an outdated income-tier estimate of $1,200 for two children under six instead of the $2,000 NJ CTC for four qualifying dependent children. It also failed to separately include the $50 renter Property Tax Credit, producing an unsupported residual above its stated components."
-us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model understated the NJ EITC by estimating the federal EITC at $7,700 instead of $8,231. It then wrongly denied the $2,000 NJ CTC and omitted the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly subjected the federal EITC to phaseout and reduced it to $6,716 rather than using the $8,231 credit that yields a $3,292.40 NJ EITC. It also applied an outdated $500 credit to only two children under six instead of the $2,000 NJ CTC for four children and omitted the $50 renter credit."
-us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model used loose and internally inconsistent estimates for both the NJ EITC and CTC instead of calculating $3,292.40 and $2,000. It also omitted the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model used an understated federal EITC of $7,787 rather than $8,231. It entirely omitted the $2,000 NJ Child Tax Credit and the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model understated the federal EITC used for the 40% NJ credit and applied an outdated $800-per-child CTC tier to only two children under six. The correct NJ CTC is $2,000 for four qualifying children, and the model also omitted the $50 renter credit."
-us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model failed to identify all three refundable New Jersey programs. The household receives a $3,292.40 NJ EITC, a $2,000 NJ CTC, and a $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model approximated the NJ EITC rather than applying 40% to the $8,231 federal EITC, and it limited the CTC to two children under six at $800 each. It missed the $2,000 credit for four qualifying children and the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model incorrectly reduced the federal EITC to $7,128 and therefore understated the NJ EITC. It also used an outdated two-child, $1,600 CTC calculation instead of $2,000 for four qualifying children and omitted the $50 renter credit."
-us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model understated both components it recognized: the NJ EITC is $3,292.40 and the NJ CTC is $2,000 for four qualifying children, not $1,600. It also omitted the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model combined the NJ EITC and CTC without applying the traced component amounts of $3,292.40 and $2,000. Its two-component total also omits the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so the required output was missing."
-us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly concluded that the facts trigger no refundable New Jersey credits. The earnings, four qualifying children, renter status, and head's age generate the NJ EITC, NJ CTC, and $50 Property Tax Credit."
-us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model treated explicit income, dependent-child, renter, and age facts as inadequate to establish refundable-credit eligibility. Those facts produce $3,292.40 of NJ EITC, $2,000 of NJ CTC, and a $50 renter credit."
-us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model correctly calculated the $3,292.40 NJ EITC and $2,000 NJ CTC but stopped at those two programs. It omitted the $50 refundable Property Tax Credit for renters with a spouse age 42 or older."
-us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model understated the federal EITC as $8,046 instead of $8,231, reducing the NJ EITC. It also omitted the entire $2,000 NJ CTC and the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model correctly used the $3,292.40 NJ EITC but applied an outdated $1,600 CTC calculation limited to two children under six. The NJ CTC is $2,000 for four qualifying children, and the $50 renter Property Tax Credit must also be included."
-us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The submitted total implies the same $3,292.40 EITC plus an outdated $1,600 CTC limited to two young children. The correct CTC is $2,000 for four qualifying children, followed by the additional $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model wrongly denied every refundable New Jersey credit. The household qualifies for the NJ EITC, NJ Child Tax Credit, and renter Property Tax Credit totaling $5,342.40."
-us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model used the wrong federal EITC amount and an outdated NJ CTC schedule of $400 for each of two children under six. The traced rules award a $2,000 CTC for four qualifying children and a separate $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model understated the federal EITC as $7,901 rather than $8,231 and thus understated the 40% NJ EITC. It also omitted the $2,000 NJ CTC and the $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model used an incorrect federal EITC of about $8,271 rather than the traced $8,231 and treated the NJ EITC as the entire state-credit total. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit."
-us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so the required output was missing."
-us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model correctly included the $3,292.40 NJ EITC and $2,000 NJ CTC. It omitted the additional $50 refundable Property Tax Credit available because the household rents and the head is age 42."
-us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly found no applicable New Jersey refundable credits. The household's earnings, qualifying children, and renter-age status produce three credits totaling $5,342.40."
-us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model wrongly treated $30,800 of earned income as eliminating the federal EITC, even though the traced federal credit is $8,231, and it incorrectly treated low tax liability and housing assistance as barriers to refundable credits. It also counted only the one-year-old for the CTC, omitting the other three qualifying children and the $50 renter credit."
-us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model invented an NJ EITC phaseout threshold and a $100 minimum-credit rule that do not govern this calculation. The household instead receives 40% of the $8,231 federal EITC, plus the $2,000 NJ CTC and $50 renter Property Tax Credit."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived the complete $8,348 calculation, then submitted $3,455 instead. Its numeric output contradicts its own taxable-income, bracket, and credit analysis."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse’s listed employer-sponsored insurance premium from gross wages even though the prompt gives $60,000 as annual gross wages and does not identify an employee pre-tax contribution. It then submitted $2,950 despite its own estimate of approximately $8,600 and without identifying credits that bridge the difference."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model’s own approximated parameters produced $8,336, only $12 from the trace calculation, but it submitted $4,880. No deduction or nonrefundable credit in its reasoning supports that reduction."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly included $25,500 of Social Security and calculated tax near $8,349, then submitted $5,732. Its answer discards its completed bracket calculation without any intervening deduction or credit."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model calculated approximately $8,300 using the correct income composition and bracket structure, then submitted $4,989. It identified no nonrefundable credit or other adjustment capable of reducing the computed tax."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used projected parameters—a $30,700 standard deduction and a $24,450 first-bracket ceiling—instead of the applicable 2026 values of $32,200 and $24,800. That inflated taxable income and produced $8,535 rather than applying the enacted 2026 parameters."
-us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included only about $16,150 of the $30,000 Social Security benefit, although the provisional-income formula reaches the 85% cap of $25,500. It then applied unspecified “minor credit adjustments” despite identifying no available nonrefundable credit, reducing its own $7,178 estimate to $6,350."
-us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the spouse’s $60,000 gross wages by the $8,139 listed employer-sponsored insurance premium. It also used a $30,600 standard deduction rather than $32,200, so both AGI and taxable income diverged from the required calculation."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $8,139 from gross wages and invented mortgage interest, state tax, and deductible medical amounts from inputs that do not supply those expenses. It also applied a TCJA-sunset regime with personal exemptions and 15% rates instead of the applicable 2026 standard deduction and 10%/12% brackets."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model failed to construct AGI under the Social Security inclusion rules and instead described an incoherent $80,360 amount that treats the standard deduction as part of gross-income computation. The correct sequence yields $105,900 of AGI and $73,700 of taxable income before applying the brackets."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model improperly netted the $8,139 employer-sponsored insurance premium from gross wages and assumed expiration-era personal exemptions and 15% brackets. The applicable 2026 calculation retains $60,000 of wages, uses the $32,200 joint standard deduction, and applies 10% and 12% rates."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a reverted pre-TCJA standard deduction, personal exemptions, and a 15% marginal bracket instead of the applicable 2026 parameters. It also improperly reduced the stated $60,000 gross wages by the employer-sponsored insurance premium."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly treated the $8,139 employer-sponsored insurance premium as a pre-tax reduction of the spouse’s stated $60,000 gross wages, lowering AGI to $97,761. Gross wages remain $60,000, so AGI is $105,900 before the $32,200 standard deduction."
-us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Although it reached the correct $105,900 AGI, the model subtracted personal exemptions and applied an unspecified 2026 bracket regime that produced $10,735. The applicable calculation uses only the $32,200 standard deduction here and taxes $73,700 at the 10% and 12% joint rates."
-us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used projected values of $30,750 for the standard deduction and $24,450 for the first-bracket ceiling. The applicable values are $32,200 and $24,800, which reduce taxable income to $73,700 and tax to $8,348."
-us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared the household below the taxable threshold without calculating taxable Social Security, AGI, or the standard deduction. The income rules produce $105,900 of AGI and $73,700 of taxable income, so the liability is positive."
-us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted $2,757 reflects unsupported health-related itemized deductions or reductions even though itemized deductions total only $3,207.96 and the $32,200 standard deduction is larger. Applying that standard deduction to $105,900 of AGI yields $73,700 of taxable income."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model stated that taxable income was $26,150, which omits a substantial portion of wages, pension income, or taxable Social Security after the standard deduction. The correct income inclusion and $32,200 standard deduction yield $73,700 of taxable income."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly found $105,900 of AGI but incorrectly applied a post-sunset $17,018 standard deduction, personal exemptions, and a 15% bracket. The applicable 2026 rules use a $32,200 joint standard deduction with 10% and 12% brackets."
-us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed pre-TCJA rules, using a $15,700 standard deduction, two personal exemptions, and a 15% marginal rate. The applicable 2026 computation uses the $32,200 joint standard deduction and 10%/12% rate schedule."
-us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the spouse’s stated $60,000 gross wages by the $8,139 employer-sponsored insurance premium. Keeping the full wages produces $105,900 of AGI and $73,700 of taxable income after the $32,200 standard deduction."
-us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly computed $105,900 of AGI but used estimated parameters of a $30,600 standard deduction and a $24,400 first-bracket ceiling. The applicable $32,200 deduction and $24,800 ceiling produce $8,348."
-us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used estimated 2026 parameters—a $30,900 standard deduction and $24,500 first-bracket ceiling—instead of $32,200 and $24,800. Those parameter errors raised taxable income and tax above the traced result."
-us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model’s stated bracket arithmetic equals $8,577, but it submitted $8,077.50 without any supporting credit or adjustment. It also used projected rather than applicable 2026 deduction and bracket parameters."
-us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model miscomputed taxable Social Security as $4,000 even though provisional income forces the $25,500 cap, and it invented a CDCC despite no dependent or care expense being listed. Its submitted $12,110.90 also contradicts its own claimed $5,930.25 pre-credit tax because subtracting a credit cannot increase liability."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model counted blindness and disability as two separate $1,000 exemptions even though the spouse receives one blind-or-disabled exemption, then improperly deducted 18% of subsidized rent. The correct $12,000 of exemptions leaves $18,915 taxable and produces $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied unspecified standard deductions, a pension exclusion, property-tax relief, and the federal American Opportunity Credit to New Jersey liability. The applicable $12,000 of NJ exemptions leaves $18,915 taxable rather than zero."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted the spouse's blindness and disability as separate exemptions and then invented nonrefundable credits that erased the remaining tax. Only one $1,000 blind-or-disabled exemption applies, and no such credit reduces the $264.81 liability."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model misapplied New Jersey's $20,000 joint-filer gross-income filing threshold as an exemption capable of eliminating tax. Gross income is $30,915, and taxable income after the applicable exemptions is $18,915, producing $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model assigned $3,000 of regular spouse exemptions and separately counted blindness, producing $13,000 instead of the correct $12,000 total exemption. That understated taxable income by $1,000 and tax by $14."
+us,scenario_008,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model counted separate $1,000 exemptions for blindness and disability and then invented a low-income phase-in that erased the tax. New Jersey allows one $1,000 blind-or-disabled exemption here, and $30,915 of gross income exceeds the $20,000 filing threshold."
+us,scenario_008,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $30,632 income figure and allowed only $9,000 of exemptions. NJ gross income is $30,915 and total exemptions are $12,000, leaving $18,915 entirely in the 1.4% bracket."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly reached $18,915 of taxable income and approximately $265 of tax, but improperly subtracted a $50 tenant property-tax credit. No such nonrefundable credit applies in this computation, so the liability remains $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated low income and personal exemptions as eliminating all taxable income. The specified exemptions total only $12,000, leaving $18,915 taxable and $264.81 of tax."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model first overstated exemptions to derive roughly $251, then improperly subtracted a $50 tenant property-tax credit. Correct exemptions are $12,000 and no renter credit reduces the resulting $264.81 tax."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model allowed $14,000 of exemptions by overstating the regular joint-filer exemption and separately counting blindness and disability. The correct exemption total is $12,000, leaving $18,915 taxable."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly computed $18,915 of taxable income and $264.81 of tentative tax, then improperly subtracted a $50 property-tax credit. The requested pre-refundable liability includes no applicable renter credit reduction."
+us,scenario_008,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model deducted $13,000 by treating blindness and disability as separate exemptions. Only one $1,000 blind-or-disabled exemption applies, so taxable income is $18,915 rather than $17,915."
+us,scenario_008,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified exemptions or credits wiped out the liability. The applicable $12,000 exemption total leaves $18,915 taxable, and no nonrefundable credit reduces the resulting $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated unspecified deductions and credits as sufficient to reduce the tax base or liability to zero. The actual exemptions leave $18,915 taxable and $264.81 due before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $181.70 of over-the-counter health expenses from NJ gross income. With only the $12,000 of applicable exemptions deducted, taxable income is $18,915 and tax is $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model incorrectly used the New Jersey child tax credit to offset this pre-refundable-credit liability. That refundable credit is excluded from the requested output, leaving the $264.81 tax unchanged."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced liability using medical expenses and a renter property-tax credit that do not enter this computation. The correct calculation deducts only $12,000 of exemptions from $30,915 and taxes $18,915 at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $250.81 answer implies $17,915 of taxable income, reflecting an extra $1,000 exemption for treating blindness and disability separately. Only one blind-or-disabled exemption applies, yielding $18,915 taxable and $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model invented a New Jersey standard deduction and unspecified credits that reduced tax to zero. New Jersey's applicable exemptions total $12,000, leaving $18,915 taxable and $264.81 due."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model substituted a federal-style $30,632 AGI and allowed only $9,000 of exemptions. NJ gross income is $30,915 and exemptions total $12,000, placing all $18,915 of taxable income in the 1.4% bracket."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly derived $18,915 of taxable income and $264.81 of tax, but improperly subtracted a $50 renter property-tax credit. No such credit reduces this liability before refundable credits."
+us,scenario_008,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted the $1,000 blind-or-disabled exemption and then taxed the entire $19,915 base at 1.75%. Total exemptions are $12,000, and the resulting $18,915 is taxed at 1.4%."
+us,scenario_008,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used only $10,000 of exemptions and improperly deducted 18% of subsidized rent. The correct calculation allows $12,000 of exemptions and no renter deduction, leaving $18,915 taxable."
+us,scenario_008,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_008,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly calculated $18,915 of taxable income and $264.81 of tentative tax, then improperly subtracted a $50 renter property-tax credit. The applicable pre-refundable liability is the unreduced $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used a federal-style $30,632 income figure and omitted the $1,000 blind-or-disabled exemption from its $11,000 exemption total. NJ gross income is $30,915 and exemptions total $12,000, leaving $18,915 taxable."
+us,scenario_008,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $181.70 of over-the-counter medical expenses and then subtracted a $50 renter property-tax credit. Neither adjustment enters this computation, so $18,915 remains taxable and produces $264.81."
+us,scenario_008,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model undervalued dependent exemptions, treated blindness and disability as potentially separate exemptions, and then invented a nonrefundable low-income reduction to zero. The correct exemptions total $12,000, and the $20,000 filing threshold does not erase tax on $30,915 of gross income."
+us,scenario_008,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that exemptions and deductions reduce New Jersey taxable income to zero. They total $12,000 against $30,915 of gross income, leaving $18,915 taxable and $264.81 of liability."
+us,scenario_008,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly phased the federal EITC down to $6,110 instead of using the $8,231 credit, and it limited the NJ CTC to two children under six rather than the four qualifying dependents producing $2,000. It also omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that New Jersey has no refundable credits. It omitted the $3,292.40 NJ EITC, $2,000 NJ CTC, and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $9,492 does not follow from the model's own stated estimates of roughly $2,880 of NJ EITC and $1,000 of NJ CTC. It also used the wrong federal EITC amount, counted only one NJ CTC child, and omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model got the $3,292.40 NJ EITC essentially right but assigned only about $1,200 of NJ CTC based on two children under six instead of $2,000 for four qualifying dependents. It also failed to add the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model understated the NJ EITC by using an approximate $7,700 federal credit instead of $8,231. It then wrongly treated the NJ CTC as inapplicable and omitted both its $2,000 value and the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied obsolete or mismatched federal EITC phaseout parameters to reduce the federal credit to $6,716 instead of $8,231. It also limited the NJ CTC to two under-six children at $500 each rather than the four qualifying dependents yielding $2,000, and omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used internally inconsistent approximations, including an NJ EITC described first as $3,400–$3,600 and then as $3,800. The exact components are $3,292.40 of NJ EITC, $2,000 of NJ CTC for four qualifying dependents, and a $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model used $7,787 instead of $8,231 as the federal EITC base, understating the NJ EITC. It entirely omitted the $2,000 NJ CTC and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model understated the NJ EITC by using a $7,691 federal credit and restricted the NJ CTC to two children under six. The NJ CTC is $2,000 for four qualifying dependents, and the calculation also requires the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model failed to identify all three applicable New Jersey refundable credits. They total $3,292.40 of NJ EITC, $2,000 of NJ CTC, and $50 of renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model approximated the NJ EITC rather than applying 40% to the $8,231 federal EITC, and it awarded only $1,600 of NJ CTC for two under-six children instead of $2,000 for four qualifying dependents. It also omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model wrongly reduced the federal EITC to $7,128, producing only $2,851 of NJ EITC. It also used $1,600 for two under-six children instead of the $2,000 NJ CTC for four qualifying dependents and omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model understated the NJ EITC at $3,219.60 rather than $3,292.40 and treated the NJ CTC as $1,600 rather than $2,000 for four qualifying dependents. It omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The total implies that the model did not apply the exact $3,292.40 NJ EITC, $2,000 NJ CTC for four qualifying dependents, and $50 renter Property Tax Credit. Those three components sum to $5,342.40, not $4,776."
+us,scenario_008,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, so the required output could not be parsed."
+us,scenario_008,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly concluded that no refundable New Jersey credit was indicated. It omitted the $3,292.40 NJ EITC, $2,000 NJ CTC, and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model treated the facts as lacking refundable-credit eligibility even though the income, dependents, renter status, and age establish three New Jersey credits. It omitted $3,292.40 of NJ EITC, $2,000 of NJ CTC, and the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model correctly included the $3,292.40 NJ EITC and $2,000 NJ CTC but stopped at $5,292.40. It omitted the $50 New Jersey Property Tax Credit available to this renter household because the head is age 42."
+us,scenario_008,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model used $8,046 rather than $8,231 for the federal EITC base and included no other state credits. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model correctly calculated the $3,292.40 NJ EITC but awarded only $1,600 of NJ CTC instead of $2,000 for four qualifying dependents. It also omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The submitted total is consistent with combining the correct $3,292.40 NJ EITC with only $1,600 of NJ CTC. The NJ CTC is $2,000 for four qualifying dependents, and the model also omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model incorrectly stated that no state refundable credits apply. It omitted the $3,292.40 NJ EITC, $2,000 NJ CTC, and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model used the wrong federal EITC base and limited the NJ CTC to $800 for two under-six children in an asserted income bracket. The applicable NJ CTC is $2,000 for four qualifying dependents, and the $50 renter Property Tax Credit must also be included."
+us,scenario_008,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model approximated the NJ EITC at $3,300 and awarded only $1,000 of NJ CTC for two under-six children. The exact amounts are $3,292.40 of NJ EITC and $2,000 of NJ CTC for four qualifying dependents, plus the omitted $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model used $7,901 rather than $8,231 as the federal EITC base and then treated the NJ EITC as the only refundable state credit. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model used an incorrect approximate federal EITC of $8,271 and included only the resulting NJ EITC. It omitted the $2,000 NJ CTC and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, so the required output could not be parsed."
+us,scenario_008,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model correctly included $3,292.40 of NJ EITC and $2,000 of NJ CTC, reaching $5,292.40. It omitted the $50 renter Property Tax Credit triggered by renter status and a spouse age 42 or older."
+us,scenario_008,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly concluded that no New Jersey refundable credits apply. It omitted the $3,292.40 NJ EITC, $2,000 NJ CTC, and $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model used an incorrect $8,262 federal EITC base and invented a phase-down of the NJ CTC based on income exceeding $30,000. The correct components are a $3,292.40 NJ EITC, a $2,000 NJ CTC for four qualifying dependents, and a $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly treated roughly $30,800 of earnings as phasing the federal EITC to zero, when the federal EITC is $8,231 and produces a $3,292.40 NJ EITC. It also counted only the one-year-old for the NJ CTC, wrongly treated housing assistance and zero tax liability as barriers to refundable credits, and omitted the $50 renter Property Tax Credit."
+us,scenario_008,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly declared the household above the NJ EITC phaseout threshold and substituted a nonexistent $100 minimum credit. The NJ EITC is $3,292.40, and the calculation also requires the $2,000 NJ CTC and $50 renter Property Tax Credit."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $105,900 of AGI, $73,700 of taxable income, and $8,348 of tax, but then submitted $3,455 instead. Its numeric output discarded its own completed calculation."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the spouse's $8,139 employer-sponsored insurance premium from the explicitly stated $60,000 of gross wages and treated all $30,000 of Social Security as AGI rather than applying the taxable-benefit formula. It then reduced its own approximate $8,600 tax calculation to $2,950 without any credit or computation supporting that reduction."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's stated inputs produced $8,336 using its slightly incorrect $32,300 deduction, yet it submitted $4,880. The submitted value does not follow from its own bracket calculation; using the exact $32,200 deduction produces $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly calculated approximately $8,349 after correctly including $25,500 of taxable Social Security, but submitted $5,732. No deduction, bracket adjustment, or credit in its reasoning supports the submitted reduction."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly calculated approximately $8,300, then submitted $4,989 without identifying any nonrefundable credit or further deduction. Its submitted value contradicts its own income and bracket computation."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used projected parameters—a $30,700 standard deduction and a $24,450 top of the 10% bracket—instead of the applicable 2026 values of $32,200 and $24,800. Those parameter errors raised taxable income and produced $8,535 instead of applying the exact brackets to $73,700."
+us,scenario_009,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included only about $16,150 of Social Security even though its own reasoning recognized that the household is above the upper joint threshold, where the 85% cap makes $25,500 taxable. It then made an unsupported downward adjustment from its calculated $7,178 to $6,350 despite identifying no credit."
+us,scenario_009,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated $60,000 of gross wages by the $8,139 employer-sponsored insurance premium, lowering AGI to $97,761. It also used a $30,600 standard deduction rather than $32,200, so it taxed the wrong income base."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly netted the $8,139 ESI premium from gross wages, invented mortgage interest from the mortgage balance, and manufactured deductible state-tax and medical amounts. It also applied personal exemptions and 10%/15% sunset brackets that do not govern this 2026 computation instead of the $32,200 standard deduction and 10%/12% brackets."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's stated $80,360 base does not implement the Social Security taxable-benefit calculation or the $32,200 joint standard deduction. The correct income calculation includes $25,500 of Social Security in $105,900 of AGI and leaves $73,700 taxable."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,139 ESI premium from the stated gross wages and then assumed a TCJA expiration regime combining personal exemptions with different rates. The applicable calculation uses $105,900 of AGI, a $32,200 standard deduction, and the 2026 10%/12% joint brackets."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced gross wages by the ESI premium and used a reverted pre-TCJA standard deduction, personal exemptions, and a 15% bracket. The applicable 2026 rules instead yield $105,900 of AGI and $73,700 of taxable income under the $32,200 standard deduction."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly treated the listed $8,139 employer-sponsored insurance premium as a pre-tax reduction of the explicitly stated $60,000 gross wage amount. That lowered AGI to $97,761 rather than $105,900 and caused it to tax too little income."
+us,scenario_009,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model added personal exemptions to its 2026 deduction treatment even though the applicable computation uses the $32,200 joint standard deduction without personal exemptions. Applying the correct deduction to its correctly stated $105,900 AGI leaves $73,700 taxable and produces $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly computed AGI and taxable Social Security but substituted projected parameters: a $30,750 standard deduction and a $24,450 bracket threshold. The exact 2026 values are a $32,200 deduction and a $24,800 10% bracket ceiling, producing $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared the household below the taxable threshold without calculating its income. Wages, pension, and $25,500 of taxable Social Security produce $105,900 of AGI and $73,700 after the standard deduction, so positive federal tax is due."
+us,scenario_009,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized health deductions to reduce the liability, but itemized deductions total only $3,207.96 and are therefore displaced by the $32,200 standard deduction. With $73,700 taxable income and no nonrefundable credits, the bracket calculation yields $8,348, not $2,757."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted only $26,150 of taxable income after accounting for wages, pension, Social Security, and the standard deduction. Those inputs instead produce $105,900 of AGI and $73,700 of taxable income, so its submitted $3,922 rests on a severely understated tax base."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied a post-sunset regime with a small standard deduction, personal exemptions, and a 15% marginal bracket. The applicable 2026 computation uses the $32,200 joint standard deduction, no personal exemptions, and the 10%/12% brackets, leaving $73,700 taxable."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly calculated $105,900 of AGI but applied an inapplicable TCJA-sunset combination of a $16,550 deduction, personal exemptions, and a 15% bracket. The correct $32,200 standard deduction and 2026 10%/12% brackets produce $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used pre-TCJA rules, including a $15,700 standard deduction, personal exemptions, and a 15% bracket. The applicable 2026 rules provide a $32,200 joint standard deduction, no personal exemptions, and 10%/12% rates on the resulting $73,700."
+us,scenario_009,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the spouse's explicitly stated $60,000 gross wages by the $8,139 ESI premium. That lowered AGI from $105,900 to $97,761 and taxable income from $73,700 to about $65,561."
+us,scenario_009,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly calculated $105,900 of AGI but used estimated parameters—a $30,600 standard deduction and a $24,400 10% threshold. The exact $32,200 deduction and $24,800 threshold produce taxable income of $73,700 and tax of $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used estimated 2026 parameters of a $30,900 standard deduction and a $24,500 10% bracket ceiling. The applicable figures are $32,200 and $24,800, which reduce taxable income to $73,700 and yield $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's written bracket calculation totals $8,577, but it submitted $8,077.50 without a supporting deduction or credit. It also used projected rather than exact 2026 parameters; the exact $32,200 deduction and $24,800 threshold yield $8,348."
+us,scenario_009,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included only $4,000 of Social Security instead of the $25,500 taxable under the 85% cap and invented a $1,050 child and dependent care credit despite no dependent or care expense being listed. Its final $12,110.90 also contradicts its own claimed $5,930.25 pre-credit tax because subtracting a credit cannot increase the liability."
 us,scenario_009,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_009,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model improperly inferred SSDI and disability-based Medicare eligibility from Social Security retirement income despite the instruction that unlisted statuses are false. At age 57 with no disability, ESRD, or ALS stated, the Head does not satisfy either the age pathway or an under-65 Medicare pathway."
 us,scenario_009,head_medicare_eligible,gpt-5.4-mini,llm_error,other,False,"The model's reasoning correctly concluded that the 57-year-old Head is not Medicare eligible, but it submitted value = 1, which denotes eligible. This is an answer-value inversion: its stated conclusion required value = 0."
@@ -652,33 +685,35 @@ us,scenario_012,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibili
 us,scenario_012,child1_wic_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied the WIC income limit without first enforcing the program's categorical age restriction. A 10-year-old is outside WIC's under-five child category, so income below the limit does not confer eligibility."
 us,scenario_012,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model explicitly misidentified age 10 as within WIC's age limit. WIC child eligibility ends at age five, making this 10-year-old categorically ineligible even though household income satisfies the financial threshold."
 us,scenario_012,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $4,427 of EITC and $1,700 of refundable CTC, then submitted $7,093 by adding an unexplained $966 despite explicitly stating that no other refundable credit applies. The two stated components sum to $6,127."
-us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated the pregnant spouse and unborn fetus as qualifying children and invoked a three-child EITC schedule. Federal EITC and CTC count only the age-10 child, and the model also omitted the $1,700 refundable CTC."
-us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an understated estimated one-child EITC instead of the 2026 maximum of $4,427, even though it correctly recognized that $22,000 is below the married-joint phaseout threshold. It then made an unsupported inflation adjustment that did not equal its own component calculation."
-us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model inflated the 2026 one-child EITC maximum to $4,731. At $22,000 of joint earnings the EITC is the unphased maximum of $4,427, which combines with the $1,700 refundable CTC to produce $6,127."
-us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used $4,367 for the EITC and a $1,800 refundable-CTC cap. The applicable 2026 amounts are $4,427 and $1,700, respectively."
-us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied the full $2,000 child credit as refundable, ignoring the $1,700 refundable cap, and used stale 2025 EITC parameters. The refundable components are $4,427 of EITC and $1,700 of CTC."
-us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model repeatedly applied incompatible EITC schedules, including a phaseout at $22,000 and an invented $5,500 EITC, before submitting an unsupported rounded total. For a married joint return with one child, $22,000 remains on the EITC plateau and yields $4,427, plus $1,700 of refundable CTC."
-us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable-CTC cap and understated the one-child EITC at $4,355. The 2026 amounts are a $1,700 refundable CTC and a $4,427 EITC."
-us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an obsolete $1,000 refundable-CTC cap. Its EITC was also $4 below the applicable $4,427 maximum; the correct components total $6,127."
-us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model overstated the one-child EITC as $5,571 and used an obsolete $1,000 refundable-CTC amount. The household receives the unphased $4,427 one-child EITC and $1,700 refundable CTC."
-us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model rounded the EITC to $4,500 and applied an obsolete $1,000 refundable-CTC cap. The exact 2026 components are $4,427 and $1,700."
-us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $1,000 rather than the 2026 $1,700 refundable-CTC cap and understated the EITC by $107. The applicable one-child EITC is $4,427."
-us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 instead of the 2026 amount of $4,427. Its $1,700 refundable CTC component was correct."
-us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model carried forward the $4,328 one-child EITC maximum rather than applying the 2026 maximum of $4,427. Adding the correctly identified $1,700 refundable CTC yields $6,127."
-us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model improperly reduced the one-child EITC to $2,591 even though $22,000 is below the married-joint phaseout threshold. It also treated zero pre-credit tax liability as barring the refundable CTC, when the earned-income formula produces the capped $1,700 refundable amount."
-us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly denied EITC eligibility because the spouse is age 18. A married couple with earned income and an age-10 qualifying child qualifies for the $4,427 one-child EITC, and the refundable child credit is $1,700 rather than $2,524."
-us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The prompt supplies wages, a spouse, an age-10 child, and assumes filing, so the model was wrong to treat filing status, earned income, and child eligibility as unspecified. Those facts establish a joint return with one qualifying child and refundable credits of $4,427 plus $1,700."
-us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated both 2026 parameters, using a $4,526 EITC and a $1,800 refundable-CTC cap. The applicable amounts are $4,427 and $1,700."
-us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly identified the $4,427 EITC but used a $1,800 refundable child-credit amount. The 2026 refundable cap is $1,700, producing $6,127."
-us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model subjected the EITC to a phaseout at $22,000 and reduced it to $2,479. Joint filers with one child remain below the applicable phaseout threshold at this income, so the EITC is $4,427; the model also omitted the $1,700 refundable CTC."
-us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied post-sunset parameters—an approximately $4,318 EITC and a $1,000 ACTC cap—instead of the operative 2026 values. The household receives $4,427 of EITC and a $1,700 refundable CTC."
-us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable-CTC cap and estimated the EITC at $4,436. The applicable 2026 components are exactly $1,700 and $4,427."
+us,scenario_012,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model explicitly derived $4,427 of EITC plus $1,700 of refundable CTC and even stated their correct $6,127 sum, then added an unexplained $966 despite also stating that no other refundable credit applied. Its submitted value contradicts its own component arithmetic."
+us,scenario_012,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly counted the pregnant spouse or unborn fetus as additional qualifying children and used an inapplicable head-of-household EITC figure. Federal EITC and CTC calculations recognize only the age-10 child here, and the joint return receives $4,427 of EITC plus $1,700 of refundable CTC."
+us,scenario_012,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $4,328 one-child EITC instead of the 2026 maximum of $4,427 and then submitted $6,172, which does not equal its own stated $6,028 component sum. With the $1,700 refundable CTC, the required total is $6,127."
+us,scenario_012,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model replaced the applicable 2026 one-child EITC maximum of $4,427 with an unsupported refined estimate of $4,731. The refundable CTC is $1,700, so inflating the EITC maximum produced the overstatement."
+us,scenario_012,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used both the wrong EITC amount, $4,367 instead of $4,427, and the wrong refundable CTC cap, $1,800 instead of $1,700. Applying the 2026 amounts yields $6,127."
+us,scenario_012,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model treated the entire $2,000 child tax credit as refundable, ignoring the 2026 $1,700 per-child refundable cap, and used the 2025 EITC maximum of $4,328. The applicable components are a $4,427 EITC and $1,700 refundable CTC."
+us,scenario_012,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model cycled through incompatible EITC calculations and ultimately submitted an unsupported $5,500 EITC despite correctly recognizing one qualifying child and a $1,700 refundable CTC. At $22,000 on a joint return, the one-child EITC is the $4,427 maximum."
+us,scenario_012,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap and understated the one-child EITC as $4,355. The 2026 components are $1,700 of refundable CTC and $4,427 of EITC."
+us,scenario_012,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a $1,000 refundable CTC cap instead of $1,700 and understated the EITC by $4. The correct component amounts are $4,427 and $1,700."
+us,scenario_012,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model assigned $5,571 of EITC to a one-child joint household even though the applicable maximum is $4,427, and it used a $1,000 refundable CTC instead of $1,700. Both credit parameters were misapplied."
+us,scenario_012,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used rough figures of $4,500 for EITC and $1,000 for refundable CTC rather than the applicable 2026 amounts. The exact components are $4,427 and $1,700."
+us,scenario_012,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model understated the one-child EITC as $4,320 and applied an obsolete $1,000 refundable CTC cap. The 2026 amounts are $4,427 and $1,700."
+us,scenario_012,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 rather than the 2026 maximum of $4,427. Its $1,700 refundable CTC component was correct."
+us,scenario_012,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the 2025 one-child EITC maximum of $4,328 rather than the 2026 maximum of $4,427. Its $1,700 refundable CTC component was correct."
+us,scenario_012,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly reduced the joint one-child EITC to $2,591 even though $22,000 is below the married-filing-jointly phaseout start and receives the $4,427 maximum. It also incorrectly denied the refundable CTC solely because pre-credit tax liability was zero; the refundable portion is $1,700."
+us,scenario_012,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly denied EITC eligibility because the spouse is age 18. The 40-year-old taxpayer, joint filing, earned income, and age-10 qualifying child support a $4,427 EITC, in addition to the $1,700 refundable CTC."
+us,scenario_012,federal_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model ignored the instruction to assume tax filing and the expressly listed $22,000 of wages and age-10 child. Those facts establish a joint return with one qualifying child, producing $4,427 of EITC and $1,700 of refundable CTC."
+us,scenario_012,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated the one-child EITC as $4,526 and used a $1,800 refundable CTC cap. The applicable 2026 amounts are $4,427 and $1,700."
+us,scenario_012,federal_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly used the $4,427 one-child EITC but applied a $1,800 refundable CTC cap instead of $1,700. That single cap error overstated the total by $100."
+us,scenario_012,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly subjected the $22,000 joint return to an EITC reduction and produced $2,479. This income is below the married-filing-jointly phaseout start, so the EITC is the full $4,427; the model also omitted the $1,700 refundable CTC."
+us,scenario_012,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an obsolete post-sunset framework, limiting refundable CTC to $1,000 and using $4,318 for EITC. The applicable 2026 amounts are a $1,700 refundable CTC and $4,427 EITC."
+us,scenario_012,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC maximum and overstated the EITC as $4,436. The correct 2026 component amounts are $1,700 and $4,427."
+us,scenario_012,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model projected the EITC maximum from 2025 to $4,436 instead of using the 2026 amount of $4,427, and it imposed an obsolete $1,000 refundable CTC cap. The refundable CTC is $1,700."
 us,scenario_012,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for federal_refundable_credits.
-us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used $4,401 rather than the 2026 one-child EITC maximum of $4,427. Its $1,700 refundable CTC calculation was correct, so the EITC understatement accounts for the $26 gap."
-us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model set the total to zero despite acknowledging both EITC and refundable CTC, incorrectly describing refundable credits as offsetting one another. Refundable credits add together: $4,427 of EITC plus $1,700 of refundable CTC."
-us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model counted the pregnancy as a newborn second qualifying child and consequently applied two-child EITC and CTC schedules. An unborn child is not a federal qualifying child for either credit in this tax year, leaving one qualifying child and components of $4,427 and $1,700."
-us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model assigned a $6,936 EITC, exceeding the one-child maximum, and claimed the child credit was fully absorbed by regular tax. The standard deduction leaves no regular income tax to offset, while the earned-income formula provides a $1,700 refundable CTC; the EITC is $4,427."
+us,scenario_012,federal_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly applied the $1,700 refundable CTC cap but understated the 2026 one-child EITC maximum as $4,401. The applicable EITC is $4,427."
+us,scenario_012,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model set the result to zero despite acknowledging both EITC and refundable CTC, and it incorrectly described refundable credits as offsetting each other. They add together: $4,427 of EITC plus $1,700 of refundable CTC."
+us,scenario_012,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly identified the maximum-credit plateau and the $1,700 refundable CTC but used $4,425 rather than the 2026 EITC maximum of $4,427. This understated the total by $2."
+us,scenario_012,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the pregnancy as a newborn qualifying child and therefore calculated both EITC and refundable CTC for two children. An unborn child is not a federal qualifying child for these credits, leaving one qualifying child, a $4,427 EITC, and a $1,700 refundable CTC."
+us,scenario_012,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model assigned an impossible $6,936 EITC to the one-child household instead of the $4,427 maximum. It also denied the refundable CTC by claiming the nonrefundable CTC was absorbed by regular tax, even though taxable income and pre-credit regular tax are zero; the refundable portion is $1,700."
 us,scenario_012,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored SNAP receipt and therefore missed categorical eligibility for free school meals. It also failed to apply the direct income pathway: the household is at 81% of the federal poverty guideline, below the 130% free-meal threshold."
 us,scenario_012,free_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly placed $22,000 above 130% of the applicable federal poverty guideline; the engine's household-size calculation puts it at 81%. It also omitted the independent SNAP categorical-eligibility pathway, which qualifies the child for free meals regardless of that income comparison."
 us,scenario_012,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model falsely stated that Mississippi adopted ACA Medicaid expansion and applied the 138% FPL expansion-adult threshold. Mississippi has no such pathway for this head, whose Medicaid category is NONE."
@@ -693,36 +728,38 @@ us,scenario_012,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output
 us,scenario_012,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_012,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below the reduced-price ceiling as sufficient for reduced-price eligibility without first assigning the household to the free-meals tier. The household's 0.81 FPG ratio and categorical eligibility produce FREE status, which makes reduced-price eligibility false."
 us,scenario_012,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly stated that income below 130% FPG places the child in the free-meal range and that reduced-price support applies only from 130% through 185%, but then submitted 1 contrary to its own derivation. FREE-tier classification, independently reinforced by categorical eligibility, makes the reduced-price output 0."
-us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly converted the $411 annual SPM energy subsidy into entitlement to a roughly $400 monthly standard utility allowance and therefore invented an excess-shelter deduction. It also introduced a contradictory $1,023 maximum allotment before abandoning its own $713 monthly calculation for an unsupported $378 award."
-us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the 20% earned-income deduction, used the wrong standard deduction and maximum allotment, and treated the energy subsidy as countable income. Those errors inflated net income and reduced the award to $84 per month instead of deriving the roughly $407.90 monthly allotment."
-us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"The model correctly reached an annual estimate near $4,946 from the three-person maximum and net income, then discarded that calculation and asserted $601 per month without any supporting computation. Its submitted $7,212 is an arithmetic override of its own correctly structured derivation."
-us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"The model's stated formula produced about $406 monthly and $4,872 annually, but it replaced that result with an unsupported $514 monthly 'typical' award. The submitted $6,168 does not follow from any deduction or allotment in its reasoning."
-us,scenario_012,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model assigned an unsupported $378 monthly benefit after only a qualitative deduction calculation. Applying the $785 maximum and the 30% contribution to net income near $1,257.67 yields about $407.90 monthly, with annual parameter variation producing $4,952.09."
-us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the pregnant spouse's unborn child as a fourth SNAP household member, raising both the maximum allotment and income limits. It also invented a monthly utility allowance from the energy-subsidy input, and its submitted $5,136 contradicts every annual total stated in its reasoning."
-us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated pregnancy as adding a fourth member to the SNAP assistance unit and therefore used a four-person maximum allotment. It also applied an unsupported utility-based shelter deduction, compounding the overstatement."
-us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $500 monthly standard utility allowance and deducted $316.67 of excess shelter costs, reducing net income to $950. The traced net income is about $1,257.67, so the invented shelter deduction overstated the monthly benefit by roughly $92."
-us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly applied a standard utility allowance triggered by the energy subsidy, producing an implied $482 monthly benefit. The computation trace instead uses net income near $1,257.67 and a benefit near $407.90 before annual aggregation."
-us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted a $446 monthly award without deriving the applicable deductions or household contribution. The $785 maximum less 30% of net income near $1,257.67 produces about $407.90, not $446."
-us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction based on rent and a standard utility allowance that is not part of the traced computation. That lowered its implied net income and inflated the annual benefit to $5,970."
-us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model included a shelter deduction and obtained an unsupported $457 monthly allotment. With traced net income of about $1,257.67, the 30% contribution leaves approximately $407.90 from the $785 maximum."
-us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $453 monthly result reflects an excess-shelter deduction not present in the engine trace. The applicable earned-income and standard deductions leave net income near $1,257.67 and an initial monthly award near $407.90."
-us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model stated that shelter costs reduced net income and consequently produced an implied monthly benefit of about $471.83. The traced deduction sequence leaves net income near $1,257.67, yielding approximately $407.90 per month before annual parameter updates."
-us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly added an unborn child to create a four-person SNAP unit and used a four-person maximum allotment. It also invented a $360 monthly utility allowance and excess-shelter deduction, causing a large overstatement."
-us,scenario_012,snap,gpt-5.4-mini,llm_error,other,False,"The model supplied only a rough $497 monthly estimate and did not calculate the 30% contribution from net income. The traced formula yields about $407.90 monthly and $4,952.09 after aggregating the year's monthly allotments."
-us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required separate SNAP eligibility flags despite the prompt supplying household composition, income, assets, and assuming take-up. The household passes the gross-income, net-income, asset, work, and immigration tests and therefore receives a positive allotment."
-us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the energy subsidy as triggering a heating/cooling utility allowance and an excess-shelter deduction, reducing net income to about $1,000. The traced net income is about $1,257.67, so its $483 monthly estimate overstates the benefit."
-us,scenario_012,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model's implied $408 constant monthly estimate is close to the initial monthly calculation but simply annualized it as $4,896. It failed to aggregate the slight later-month changes in standard deductions and poverty-guideline parameters that raise the annual total to $4,952.09."
-us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied heating/cooling utility and shelter deductions that lowered its implied net income enough to produce $478 per month. The traced deduction sequence instead yields net income near $1,257.67 and a monthly award near $407.90."
-us,scenario_012,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model essentially reproduced the initial monthly award as $408 but multiplied that fixed amount by 12. It omitted the later-month parameter changes included in PolicyEngine's monthly aggregation, which produce $4,952.09 for the year."
-us,scenario_012,snap,grok-4.3,llm_error,other,False,"The model gave an unsupported approximation of $300 per month without identifying the maximum allotment, net income, or 30% contribution. The applicable calculation produces roughly $407.90 monthly before annual aggregation."
-us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used outdated values for both the standard deduction and three-person maximum allotment, specifically $198 and $766 instead of the traced parameters around $209 and $785. Those parameter errors reduced its calculated monthly benefit to $385."
-us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model tested excess shelter costs on annual figures, which is structurally valid only if every monthly parameter is annualized consistently, but then used projected and inaccurate annual deduction and maximum-allotment values. It also ignored the within-year parameter updates reflected in the $4,952.09 aggregation."
-us,scenario_012,snap,inkling,llm_error,period_annualization,False,"The model approximated annual net income as $15,080 and then rounded the monthly benefit to $407 before multiplying by 12. It omitted the precise monthly computation and later-month parameter changes, losing $68.09 from the annual total."
-us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, so it failed the required output contract before any substantive SNAP calculation could be evaluated."
-us,scenario_012,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly identified monthly net income near $1,257.67 and a benefit near $407.70, but treated that single monthly result as constant for all twelve months. It omitted the later-month standard-deduction and poverty-guideline updates included in the annual total."
-us,scenario_012,snap,minimax-m3,llm_error,other,False,"The model stopped at a note that SNAP still needed to be computed and then submitted zero. Completing the eligibility tests and allotment formula yields a positive annual benefit of $4,952.09."
-us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $411 SPM energy subsidy as unearned SNAP income and used outdated $198 and $768 monthly parameters. It then submitted $5,111 even though its own displayed annual formula produced about $4,525, so the final value also lacks computational support."
-us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $22,000 of earnings as SNAP net income and omitted both the 20% earned-income deduction and the standard deduction. Those deductions reduce monthly net income to about $1,257.67, below the net-income limit and sufficient for a positive allotment."
+us,scenario_012,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly converted the $411 annual SPM energy-subsidy input into qualification for a roughly $400 monthly utility allowance and an excess-shelter deduction. It also introduced a contradictory $1,023 maximum allotment before abandoning its own $713 monthly calculation for an unsupported $378 estimate."
+us,scenario_012,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the 20% earned-income deduction, used incorrect standard-deduction and maximum-allotment parameters, and treated the energy subsidy as countable income. Those errors inflated net income and reduced the benefit to $84 per month instead of applying 30% to $1,257.67 and subtracting it from $785."
+us,scenario_012,snap,claude-opus-4.7,llm_error,other,False,"The model's initial three-person derivation produced about $4,946, but it discarded that calculation and submitted $7,212 without a supporting computation. The submitted value is inconsistent with its own stated $785 maximum allotment and $1,242.66 net income."
+us,scenario_012,snap,claude-opus-4.8,llm_error,other,False,"The model correctly derived a benefit near $406 per month, then replaced it with an unsupported $514 monthly estimate. It also failed to aggregate the actual monthly values across the year, including the later parameter updates that produce $4,952.09."
+us,scenario_012,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model reduced the calculated monthly allotment to $378 without identifying any applicable rule or parameter supporting that amount. Using the three-person $785 maximum and 30% of $1,257.67 produces about $407.90 in the initial months, followed by the traced annual aggregation."
+us,scenario_012,snap,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the pregnant spouse as two people; an unborn child is not an additional SNAP household member. It compounded that error by applying a four-person maximum allotment and an unsupported utility-based excess-shelter deduction, and its submitted $5,136 also contradicted the $7,968 conclusion in its explanation."
+us,scenario_012,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,The model incorrectly treated pregnancy as adding a fourth SNAP member and therefore used a four-person maximum allotment. It also applied an excess-shelter deduction from a utility allowance not used in the traced net-income calculation.
+us,scenario_012,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $500 monthly standard utility allowance from the energy-subsidy input and deducted $316.67 of excess shelter. The applicable deductions leave $1,257.67 of net income, not $950, so its household contribution was understated."
+us,scenario_012,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The answer explicitly relied on a standard utility allowance triggered by the energy subsidy, creating an excess-shelter deduction absent from the traced calculation. Its $482 monthly result therefore understates countable net income relative to $1,257.67."
+us,scenario_012,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $446 monthly allotment exceeds the amount obtained by subtracting 30% of $1,257.67 from the $785 three-person maximum. The answer did not apply the controlling maximum-allotment and household-contribution calculation."
+us,scenario_012,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction based on rent and a standard utility allowance, reducing net income below the traced $1,257.67. That unsupported deduction inflated the annual SNAP benefit."
+us,scenario_012,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model's $457 monthly estimate reflects an excess-shelter deduction that is not part of the traced net-income calculation. Applying the earned-income and standard deductions yields $1,257.67 of net income and an initial monthly allotment near $407.90."
+us,scenario_012,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction and thereby reduced net income below $1,257.67. Its resulting annual allotment is inflated relative to the calculation using only the applicable earned-income and standard deductions."
+us,scenario_012,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer's $5,662 total implies a monthly allotment above the amount produced from $1,257.67 of net income. Its reference to shelter costs shows that it reduced net income with a shelter deduction absent from the traced calculation."
+us,scenario_012,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly added an unborn child to the SNAP unit, changing the household from three to four and using a four-person maximum allotment. It also invented a $360 monthly utility allowance and deducted excess shelter, further inflating the benefit."
+us,scenario_012,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer's roughly $497 monthly estimate exceeds the allotment derived from the three-person maximum and $1,257.67 net income. Its reliance on housing and utility costs implies an unsupported shelter deduction that lowered countable income."
+us,scenario_012,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required separate SNAP eligibility flags despite the prompt supplying the household composition, income, assets, and take-up assumption needed to determine eligibility. The household passes the gross-income, net-income, asset, work, and immigration tests and therefore does not receive zero."
+us,scenario_012,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the energy subsidy as triggering a heating-and-cooling utility allowance and an excess-shelter deduction. That reduced its estimated net income to about $1,000 instead of the traced $1,257.67 and inflated the monthly benefit to $483."
+us,scenario_012,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model used a single estimated monthly calculation and returned $4,896 for the year. The annual output must aggregate all twelve monthly allotments, including later-month changes to the standard deduction and poverty parameters, which produces $4,952.09."
+us,scenario_012,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied heating-and-cooling utility and shelter deductions that reduced net income below $1,257.67. That produced an inflated $478 monthly allotment instead of the amount based on the traced deductions."
+us,scenario_012,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model multiplied a rounded $408 monthly estimate by 12 and ignored the month-specific parameter updates. The annual SNAP variable is the sum of the monthly allotments, yielding $4,952.09 rather than $4,896."
+us,scenario_012,snap,grok-4.3,llm_error,other,False,"The model supplied an unsupported $300 monthly approximation without calculating the three-person maximum allotment or the 30% contribution from $1,257.67 of net income. That shortcut understated the benefit by more than $1,300 annually."
+us,scenario_012,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used obsolete $198 standard-deduction and $766 maximum-allotment amounts instead of the traced 2026 parameters, including the $785 three-person maximum. It also annualized one rounded monthly result rather than aggregating month-specific allotments."
+us,scenario_012,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a LIHEAP-triggered standard utility allowance and excess-shelter deduction, lowering net income below $1,257.67. That unsupported deduction inflated the monthly allotment to $456."
+us,scenario_012,snap,grok-build-0.1,llm_error,period_annualization,False,"The model performed SNAP's monthly excess-shelter comparison on annual totals, comparing annual rent with half of annual adjusted income. It also used an inaccurate annual maximum and omitted the month-specific parameter changes needed to sum the 2026 benefit."
+us,scenario_012,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model stated annual net income of about $15,080, which corresponds closely to the traced $1,257.67 monthly net income, but then rounded the monthly benefit to $407 and multiplied by 12. That loses the precise monthly calculation and the later-month parameter updates that raise the annual total to $4,952.09."
+us,scenario_012,snap,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric SNAP output or explanation. It therefore failed the required output contract before any substantive SNAP calculation could be evaluated.
+us,scenario_012,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly reached net monthly income of $1,257.67 but used an imprecise $407.70 monthly benefit and multiplied it uniformly by 12. The calculation requires the precise 30% contribution and aggregation of month-specific allotments, including later parameter updates."
+us,scenario_012,snap,minimax-m3,llm_error,missing_output,False,"The model acknowledged that SNAP required computation but submitted zero without performing it. The household passes every eligibility test, and the maximum-allotment-minus-contribution formula produces a positive benefit in every month."
+us,scenario_012,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model invented a roughly $470 monthly utility allowance from the energy-subsidy input and used it to deduct excess shelter. This lowered net income to about $985 instead of $1,257.67 and inflated the benefit to about $487 per month."
+us,scenario_012,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $411 SPM energy subsidy as unearned SNAP income and used outdated $768 maximum-allotment and $198 standard-deduction parameters. It then submitted $5,111 despite its own arithmetic producing $4,525, leaving the final value unsupported."
+us,scenario_012,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $22,000 of wages as SNAP net income and omitted both the 20% earned-income deduction and the standard deduction. Those deductions reduce net income to $1,257.67 per month, below the net-income limit and sufficient for a positive allotment."
 us,scenario_012,spouse_chip_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated pregnancy and income below a pregnancy-coverage threshold as sufficient for CHIP. It omitted the prior Medicaid screen: the 18-year-old spouse qualifies for Medicaid under the OLDER_CHILD category, which makes her ineligible for CHIP."
 us,scenario_012,spouse_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that pregnancy creates CHIP eligibility regardless of age once income requirements are met. It failed to apply the rule excluding anyone already Medicaid-eligible; this spouse is Medicaid-eligible as an OLDER_CHILD.
 us,scenario_012,spouse_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified low-income pregnancy coverage as CHIP-funded coverage without first testing Medicaid eligibility. The spouse's Medicaid eligibility under the OLDER_CHILD category precludes CHIP, irrespective of household size adjustments for the unborn child or pregnancy income limits."
@@ -733,68 +770,71 @@ us,scenario_012,spouse_chip_eligible,qwen-3.7-max,llm_error,categorical_eligibil
 us,scenario_012,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to classify the 18-year-old spouse under Mississippi's OLDER_CHILD Medicaid category and therefore never applied that category's MAGI income test. At 0.67 times FPL, the spouse satisfies the income condition and is eligible."
 us,scenario_012,spouse_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the spouse's explicitly listed pregnancy, which places her in a WIC categorical group, and failed to apply the WIC income limit to the household's $22,000 income. Those facts establish her eligibility; the presence of a young child is not needed for her pregnancy-based pathway."
 us,scenario_012,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_012,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated $3,900 as final Mississippi taxable income and applied 4% directly, stopping before the unit-level deductions and exemptions reduce the household's preliminary individual liability to zero."
-us,scenario_012,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model imposed a 3.9% rate on its independently constructed $3,900 taxable-income base. It failed to carry the preliminary individual calculation into Mississippi's unit-level calculation, where the applicable household deductions and exemptions eliminate the liability."
-us,scenario_012,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used only a $6,000 married personal exemption, leaving $11,400 taxable, and then applied an incorrect 4.4% rate. This understates the household-level exemptions and bypasses the unit-level calculation that reduces the preliminary Mississippi liability to zero."
-us,scenario_012,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model invented $2,300 exemptions for each household member and therefore left $10,500 taxable. Mississippi's unit-level deductions and exemptions eliminate the preliminary individual liability instead of producing the model's $420 tax."
-us,scenario_012,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model's own deduction-and-exemption calculation reached zero taxable income and zero tax, but it discarded that result and submitted an unsupported $340 based on an arbitrary $6,800 taxable-income estimate. It also used an incorrect 5% rate rather than following the 2026 unit-level computation."
-us,scenario_012,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied an obsolete and internally invalid rate schedule—claiming 5% on the first $10,000 and 3% above it—and invented a $970 family credit. It also used the wrong standard deduction and exemption amounts, preventing the unit-level deductions and exemptions from reducing liability to zero."
+us,scenario_012,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model stopped at a reconstructed $3,900 taxable-income calculation and applied 4%, producing $156. It omitted the Mississippi unit-level computation that reduces the $118 individual-level liability to $0."
+us,scenario_012,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied a 3.9% rate directly to its reconstructed $3,900 taxable income. That shortcut bypassed the Mississippi unit-level calculation that reduces the individual-level $118 liability to $0."
+us,scenario_012,state_income_tax_before_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model independently reconstructed Mississippi taxable income from AGI, the joint standard deduction, and exemptions, then treated 4% of that amount as the final household liability. It failed to carry the result through the Mississippi unit-level calculation, where the $118 individual liability is reduced to $0."
+us,scenario_012,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an incorrect $6,000 married personal exemption and a 4.4% rate, inflating taxable income to $11,400 and tax to about $502. It also omitted the Mississippi unit-level reduction that makes the benchmark output $0."
+us,scenario_012,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly assigned a $2,300 personal exemption to each household member and treated $10,500 as Mississippi taxable income. It then stopped at a simple 4% calculation instead of applying the unit-level computation that reduces liability to $0."
+us,scenario_012,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model’s own deductions repeatedly produced $0 taxable income and $0 tax, but it discarded that result and invented a $6,800 taxable-income estimate solely to report $340. It neither followed its arithmetic nor applied the Mississippi unit-level calculation that yields $0."
+us,scenario_012,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an incorrect $7,400 standard deduction, invented a two-rate calculation that charges $1,000 on the first $10,000, and subtracted an unsupported $970 family credit. Those rules do not reproduce the Mississippi computation, whose unit-level step reduces the $118 individual liability to $0."
 us,scenario_013,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age and disability as establishing an SSI-related or medically needy Arizona Medicaid pathway without identifying or applying an actual qualifying category. The head receives no SSI and qualifies through none of the available pathways, so assigning eligibility from an asserted aged/disabled income standard was incorrect."
 us,scenario_013,head_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model improperly applied ALTCS's 300%-of-SSI income ceiling without establishing eligibility for the institutional long-term-care pathway, then ignored that pathway's $2,000 resource limit despite $58,700 in bank assets. It also invented a large Social Security exclusion and medical-expense spend-down to force income below a regular aged/disabled limit; those calculations do not place the head in any Arizona Medicaid category."
 us,scenario_013,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model applied Arizona’s 2.5% rate to $88 of dividends and capital gains without subtracting the Arizona standard deduction, which reduces taxable income to zero. It also violated its own computation by submitting $744.10 after its explanation calculated $2.20."
-us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Arizona offers no refundable individual income-tax credits. It omitted the refundable increased excise tax credit, which awards this eligible low-income, one-person tax unit $25."
-us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model limited Arizona refundable credits to programs requiring dependents, rent, or property-tax payments. The increased excise tax credit requires none of those facts and provides $25 based on the household’s $6,736 adjusted gross income and tax-unit size of one."
-us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly attached a rent or property-tax requirement to the increased excise tax credit despite computing very low Arizona income. With adjusted gross income of $6,736, the one-person tax unit satisfies the credit’s income test and receives $25."
-us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model incorrectly treated dependents or positive tax liability as necessary for Arizona’s increased excise tax credit. This refundable credit pays an eligible single-person tax unit $25 even with no dependents and no pre-credit state tax liability.
-us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model correctly identified the $25 increased excise tax credit but incorrectly counted age and disability as two additional credit units. PolicyEngine applies one $25 amount for this tax-unit member; age 65+ and disability do not raise the credit to three times that amount.
-us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model omitted Arizona’s refundable increased excise tax credit and focused on dependents and property-tax programs. The household qualifies through its $6,736 adjusted gross income and one-person tax-unit size, producing $25."
-us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated dependents as necessary for Arizona refundable-credit eligibility. A taxpayer without dependents can receive the increased excise tax credit, and this qualifying one-person unit receives $25."
-us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The answer omitted the Arizona increased excise tax credit. The resident’s $6,736 adjusted gross income and tax-unit size of one satisfy its eligibility rules and yield $25."
-us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model incorrectly concluded that no state-specific refundable credit applied. Arizona’s increased excise tax credit applies to this low-income one-person tax unit and equals $25.
-us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model failed to apply the Arizona increased excise tax credit. The household meets the residence and low-adjusted-gross-income conditions, so its one eligible tax-unit member generates $25."
-us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model omitted the refundable increased excise tax credit from Arizona’s applicable programs. Adjusted gross income of $6,736 qualifies this one-person unit for the full $25."
-us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly concluded that household income exceeded Arizona’s refundable-credit limit. The relevant adjusted gross income is $6,736, which satisfies the increased excise tax credit’s income test and yields $25."
-us,scenario_013,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly stated that Arizona had no refundable credit available to this household. It omitted the increased excise tax credit, under which the qualifying one-person tax unit receives $25."
-us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize that the household facts trigger Arizona’s increased excise tax credit. Arizona residence, $6,736 of adjusted gross income, and a tax-unit size of one produce a $25 refundable credit."
-us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required dependents or earned income for a qualifying Arizona refundable credit. The increased excise tax credit requires neither for this household and provides $25 under its low-income test.
-us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted the applicable Arizona increased excise tax credit. The reported facts establish eligibility through Arizona residence and $6,736 of adjusted gross income, yielding $25 for one tax-unit member."
-us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model wrongly treated property tax, rent, dependents, or another special expense as necessary for every Arizona refundable credit. The increased excise tax credit uses the household’s qualifying adjusted gross income and one-person unit size, producing $25."
-us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model failed to apply the increased excise tax credit to an elderly one-person household. Age does not disqualify the taxpayer, and adjusted gross income of $6,736 qualifies the unit for $25."
-us,scenario_013,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model evaluated an inapplicable working-poor credit and overlooked Arizona’s increased excise tax credit. That credit does not require earnings and provides this low-income one-person unit $25.
-us,scenario_013,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model focused on the property-tax credit and incorrectly treated property-tax expenses as necessary for all Arizona refundable credits. The separate increased excise tax credit requires no property-tax payment and yields $25 here.
-us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model omitted the Arizona increased excise tax credit when assessing the household’s composition and income. A one-person tax unit with $6,736 of adjusted gross income qualifies for its full $25 amount."
-us,scenario_013,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model incorrectly treated taxable income, dependents, or separately listed program take-up as prerequisites for Arizona refundable credits. The increased excise tax credit is available based on Arizona residence and qualifying adjusted gross income, producing $25."
-us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model restricted refundable credits to earned-income or property-tax pathways. Arizona’s increased excise tax credit requires neither earned income nor a qualifying expense, and this low-income one-person unit receives $25."
-us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model compared total cash income of $30,544 with the increased excise tax credit’s adjusted-gross-income limit. The applicable adjusted gross income is $6,736 after the relevant treatment of Social Security income, so the unit qualifies for $25."
-us,scenario_013,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of dependents as disqualifying. Arizona’s increased excise tax credit covers an eligible taxpayer without dependents, and this one-person unit receives $25."
-us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model considered child, earned-income, and family-credit pathways but omitted the increased excise tax credit. That credit does not require children or earnings, and the household’s $6,736 adjusted gross income qualifies it for $25."
-us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of state tax liability as preventing a refundable credit. Arizona’s increased excise tax credit is refundable and provides the eligible one-person tax unit $25 regardless of liability available to offset.
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model improperly subtracted $14,717 of employer-sponsored insurance premiums from the supplied gross wages, even though the gross-wage input remains $88,927.65 in the federal AGI computation. Its submitted $5,647 also contradicts its own resulting tax calculation of about $4,545."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $28,000 joint standard deduction instead of $32,200 and then incorrectly applied 12% to all taxable income rather than taxing the first bracket at 10%. Its unexplained adjustment to $6,819 does not follow from its stated arithmetic."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived taxable income near $56,728 and regular tax near $6,324 using approximate brackets, then submitted $5,235 without any deduction, credit, or bracket computation supporting that reduction."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's reasoning reached approximately $6,311 from the correct income, deduction, and bracket structure, but it submitted $7,466. No stated tax component or nonrefundable-credit step produces the submitted amount."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed approximately $6,311 using the $32,200 standard deduction and the 10% and 12% brackets, then replaced that result with $6,712 without supporting arithmetic."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a projected $30,000 standard deduction instead of the 2026 $32,200 amount, leaving taxable income $2,200 too high. It also relied on 2025 or projected bracket thresholds instead of the applicable 2026 parameters."
-us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated bracket calculation produced about $6,279, close to the correct method, but it submitted $8,079 without identifying any additional tax. The unexplained $1,800 increase is not part of federal income tax before refundable credits."
-us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired and applied a $15,400 deduction, personal exemptions, and 10%/15% brackets. It also deducted the separately listed $14,717 insurance-premium input from gross wages, whereas the computation uses $88,927.65 of AGI and the 2026 $32,200 standard deduction."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset regime with personal exemptions and 10%/15% brackets instead of the applicable 2026 $32,200 joint deduction and 10%/12% brackets. It additionally reduced the supplied gross wages by the separate employer-premium amount."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $13,915 is inconsistent with taxing $56,727.65 of joint taxable income through the 2026 10% and 12% brackets. The correct bracket computation yields $6,311.32, and no additional tax applies."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly reduced gross wages by $14,717 and used assumed post-sunset deductions, personal exemptions, and pre-TCJA rates. The applicable computation instead uses $88,927.65 of AGI, a $32,200 standard deduction, and the 2026 10%/12% joint brackets."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly assumed TCJA expiration, used a roughly $16,000 standard deduction and a 15% second bracket, and subtracted $14,717 from the supplied gross wages. The applicable 2026 calculation uses the $32,200 deduction and 12% second bracket without that wage reduction."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model reduced the supplied gross wages by the separate $14,717 premium input and invoked personal exemptions that are absent from the applicable calculation. Tax must be computed from $56,727.65 of taxable income after the $32,200 joint standard deduction."
-us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $16,100 standard deduction, personal exemptions, and a 15% bracket while also reducing wages by employer premiums. The applicable 2026 rules instead produce $56,727.65 of taxable income and tax it at the 10% and 12% joint rates."
-us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction rather than $32,200, overstating taxable income by $2,200. Applying the 2026 joint brackets to $56,727.65 yields $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $10,458 does not result from the 2026 joint brackets applied to wages after the $32,200 standard deduction. With $56,727.65 of taxable income and no nonrefundable credits, the bracket tax is $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented nonrefundable credits that fully offset the tax, although no qualifying dependent or other credit-generating fact is listed. The standard deduction leaves $56,727.65 taxable rather than eliminating income, and no credit reduces the resulting $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied an obsolete post-TCJA-sunset framework with personal exemptions and a smaller standard deduction. The applicable 2026 regime uses a $32,200 joint standard deduction, no personal exemptions in this computation, and 10%/12% brackets on $56,727.65."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $88,927.65 of wages as fully offset by deductions and exemptions. The $32,200 standard deduction leaves $56,727.65 of taxable income, while child support is nondeductible and no exemption or credit eliminates the tax."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed a post-TCJA-sunset $17,018 deduction, personal exemptions, and a 15% second bracket. The applicable 2026 computation instead uses the $32,200 joint standard deduction and 10%/12% brackets, producing $6,311.32."
-us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $14,717 of employer-sponsored insurance premiums from the supplied gross wages and used a $30,840 deduction instead of $32,200. Federal AGI is $88,927.65, leaving $56,727.65 taxable."
-us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied pre-TCJA-style personal exemptions, a roughly $16,700 deduction, and a 15% second bracket. The applicable 2026 calculation uses a $32,200 joint standard deduction and the 10%/12% bracket schedule."
-us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric value or explanation for the requested output.
-us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the joint standard deduction and a personal exemption as reducing $88,927.65 of wages to nearly zero. The applicable $32,200 deduction leaves $56,727.65 taxable, and no personal exemption eliminates it."
-us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly included $18,480 of veterans benefits in AGI and invented two children and a $4,000 Child Tax Credit despite no children being listed. Its negative $17,600 submission also does not follow from its own stated $4,616.80 post-credit tax and is impossible for this before-refundable-credits liability."
-us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented CTC or ODC eligibility despite no dependent being listed and incorrectly said the standard deduction reduced taxable income to zero. The deduction leaves $56,727.65 taxable, and no nonrefundable credit offsets the resulting tax."
+us,scenario_013,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Arizona offers no refundable individual credits and omitted the refundable increased excise tax credit. The household's $6,736 adjusted gross income qualifies it for the $25 credit."
+us,scenario_013,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model considered dependent-, rent-, and property-tax-based credits but omitted Arizona's increased excise tax credit. That credit requires none of those facts here and awards this qualifying one-person tax unit $25."
+us,scenario_013,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly treated rent, property-tax payments, or dependents as necessary for every relevant Arizona refundable credit. The increased excise tax credit instead uses the household's qualifying $6,736 adjusted gross income and yields $25."
+us,scenario_013,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly denied the increased excise tax credit because the filer is single, has no dependents, and has no tax liability. Those conditions do not eliminate the refundable $25 credit for this qualifying low-income one-person tax unit."
+us,scenario_013,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly identified the $25 increased excise tax credit but incorrectly multiplied it by separate personal, elderly, and disability exemptions. PolicyEngine applies one $25 amount for this one-person tax unit; age and disability do not raise it to $75."
+us,scenario_013,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model omitted Arizona's refundable increased excise tax credit and focused on unrelated family and property-tax credits. The household qualifies for the former based on its $6,736 adjusted gross income and receives $25."
+us,scenario_013,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly treated having dependents as necessary for Arizona refundable-credit eligibility. A one-person tax unit qualifies for the $25 increased excise tax credit at $6,736 of adjusted gross income."
+us,scenario_013,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The zero answer omits Arizona's increased excise tax credit. Residence in Arizona and adjusted gross income of $6,736 qualify this one-person tax unit for the full $25."
+us,scenario_013,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model incorrectly stated that the household has no state-specific refundable credit. Arizona's increased excise tax credit applies and contributes $25.
+us,scenario_013,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model failed to apply Arizona's increased excise tax credit. The household's $6,736 adjusted gross income qualifies it for the full $25."
+us,scenario_013,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model omitted the refundable Arizona increased excise tax credit. This one-person household meets its income eligibility rule and receives $25.
+us,scenario_013,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model incorrectly concluded that household income exceeds the refundable-credit limit. The relevant adjusted gross income is $6,736, which remains within the increased excise tax credit limit and produces $25."
+us,scenario_013,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable Arizona credit is available. The increased excise tax credit applies to this qualifying low-income Arizona tax unit and equals $25.
+us,scenario_013,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to trigger Arizona's increased excise tax credit. The household's Arizona residence and $6,736 adjusted gross income yield a refundable $25."
+us,scenario_013,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required dependents, earned income, or another special trigger for a refundable Arizona credit. The increased excise tax credit requires none of those facts here and pays $25."
+us,scenario_013,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted Arizona's increased excise tax credit. The reported facts establish Arizona residence and qualifying adjusted gross income of $6,736, producing $25."
+us,scenario_013,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly treated property tax, rent, dependents, or similar expenses as necessary for all Arizona refundable credits. The increased excise tax credit depends on the applicable income and tax-unit rules here and equals $25."
+us,scenario_013,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly excluded this one-person elderly household from refundable-credit eligibility. Arizona's increased excise tax credit applies at its $6,736 adjusted gross income and pays $25."
+us,scenario_013,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model focused on a working-poor credit and income-source requirements while omitting the increased excise tax credit. The latter does not require earned income here and provides $25.
+us,scenario_013,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model treated the absence of property-tax payments as dispositive by focusing on the property-tax credit. Arizona's separate increased excise tax credit requires no listed property-tax expense and yields $25.
+us,scenario_013,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model failed to apply the Arizona increased excise tax credit. The one-person tax unit's $6,736 adjusted gross income qualifies it for $25."
+us,scenario_013,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model incorrectly treated taxable income, dependents, or separately listed program take-up as prerequisites for all refundable credits. The increased excise tax credit is available to this qualifying filer and contributes $25."
+us,scenario_013,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model restricted its analysis to earned-income and property-tax-style credits. Arizona's increased excise tax credit does not require earned income or a qualifying expense here and pays $25.
+us,scenario_013,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used $30,544 of gross income as the income tested against the increased excise tax credit limit. The applicable adjusted gross income is $6,736 after the relevant treatment of Social Security and other income, so the household receives $25."
+us,scenario_013,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of dependents as eliminating all Arizona refundable credits. This one-person tax unit qualifies for the $25 increased excise tax credit.
+us,scenario_013,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model evaluated only Arizona's property-tax refund rules and treated their income limit as controlling all refundable credits. The separate increased excise tax credit applies to the household's $6,736 adjusted gross income and yields $25."
+us,scenario_013,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model limited Arizona refundable credits to child-, earned-income-, and family-based programs. The increased excise tax credit requires neither a child nor earned income here and provides $25."
+us,scenario_013,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of state tax liability as barring the credit. Arizona's increased excise tax credit is refundable and therefore pays $25 independently of liability.
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the spouse's $14,717 employer-sponsored insurance premium from the stated annual gross wages, even though the prompt does not identify that amount as a pretax employee wage deduction. Its submitted $5,647 also contradicts its own resulting tax calculation of about $4,545."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $28,000 joint standard deduction instead of $32,200 and then incorrectly applied 12% to all taxable income rather than applying the graduated 10% and 12% brackets. Its final $6,819 is also unsupported by the $7,311 computation stated in its explanation."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly reached approximately $56,728 of taxable income and computed approximately $6,324 using its estimated brackets, but then submitted $5,235 without any valid adjustment. Applying the 2026 joint brackets to $56,727.65 yields $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's reasoning essentially reproduced the correct bracket calculation and explicitly obtained about $6,311, but it submitted $7,466. The failure is a final-answer inconsistency rather than a tax-rule error."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income near $56,728 and tax near $6,311, then replaced that result with $6,712 without a supporting computation. Its submitted value does not follow from its own stated deduction and brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the $30,000 2025 joint standard deduction instead of the 2026 amount of $32,200, leaving taxable income $2,200 too high. It also used projected or prior-year bracket thresholds rather than the applicable 2026 parameters."
+us,scenario_014,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated assumptions produced approximately $6,279 of tax, but it submitted $8,079 after an unexplained adjustment. No deduction, bracket, additional tax, or nonrefundable-credit step in its reasoning supports the extra $1,800."
+us,scenario_014,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by $14,717 for employer-sponsored insurance and applied an assumed TCJA sunset, including a $15,400 standard deduction and restored personal exemptions. The applicable computation uses the full stated wages, a $32,200 joint standard deduction, and the 2026 10% and 12% brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $14,717 of employer-sponsored insurance from gross wages and then used an assumed post-TCJA-sunset standard deduction, personal exemptions, and a 15% bracket. The correct taxable income is $56,727.65 after the $32,200 standard deduction, with no personal exemptions."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $13,915 does not result from applying the 2026 married-joint standard deduction and graduated brackets to the $88,927.65 of taxable wages. Those steps produce $56,727.65 of taxable income and $6,311.32 of tax, so the answer implies materially incorrect deductions or rates."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $14,717 employer-sponsored insurance premium from the stated wages and substituted assumed post-sunset deductions, personal exemptions, and pre-TCJA brackets. The calculation instead uses full wage AGI, the $32,200 standard deduction, and current 2026 joint brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated wages by $14,717 and applied an assumed TCJA sunset with a roughly $16,000 standard deduction and a 15% marginal bracket. No personal exemptions apply, and the 2026 joint standard deduction is $32,200."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $14,717 of employer-sponsored insurance premiums from the spouse's stated gross wages and then invoked personal exemptions. The correct calculation starts with $88,927.65 of AGI and subtracts only the $32,200 standard deduction."
+us,scenario_014,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $14,717 of employer-sponsored insurance from wages and used a $16,100 standard deduction plus $10,600 of personal exemptions. The applicable $32,200 standard deduction with no personal exemptions produces taxable income of $56,727.65, taxed under the 10% and 12% brackets rather than a 15% bracket."
+us,scenario_014,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $30,000 joint standard deduction instead of $32,200, overstating taxable income by $2,200. Applying the applicable 2026 deduction and bracket threshold produces $6,311.32 rather than $6,591.36."
+us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The answer implies that the model did not apply the 2026 married-joint standard deduction and graduated 10% and 12% brackets correctly. Full wage AGI of $88,927.65 less $32,200 yields $56,727.65 of taxable income and $6,311.32 of tax, not $10,458."
+us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model invented nonrefundable credits or deductions sufficient to eliminate the liability even though no qualifying dependents, credit expenses, or other applicable credits are listed. The standard deduction leaves $56,727.65 taxable, and the resulting $6,311.32 is not offset by any nonrefundable credit."
+us,scenario_014,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied assumed post-TCJA rules, including personal exemptions and a smaller standard deduction, rather than the applicable $32,200 joint standard deduction with no personal exemptions. This overstated taxable income and introduced a 15% bracket instead of taxing $56,727.65 under the applicable 10% and 12% brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $88,927.65 of wages as fully absorbed by deductions and exemptions. The $32,200 standard deduction leaves $56,727.65 of taxable income; child support is nondeductible and no personal exemptions or credits eliminate the tax."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model imposed assumed post-TCJA-sunset parameters, using a $17,018 standard deduction, restored personal exemptions, and a 15% bracket. The applicable rules use a $32,200 standard deduction, no personal exemptions, and the 2026 10% and 12% joint brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored personal exemptions and the smaller pre-TCJA standard deduction, leaving $61,928 taxable and applying a 15% bracket. The applicable computation leaves $56,727.65 taxable after the $32,200 standard deduction and produces $6,311.32."
+us,scenario_014,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $14,717 employer-sponsored insurance premium from the stated gross wages. Although its bracket method was otherwise close, starting from $74,211 instead of $88,927.65 reduced taxable income and tax substantially."
+us,scenario_014,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used assumed pre-TCJA rules with a roughly $16,700 standard deduction, two personal exemptions, and a 15% bracket. The applicable computation uses the $32,200 standard deduction, no personal exemptions, and the 2026 10% and 12% joint brackets."
+us,scenario_014,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not satisfy the output contract."
+us,scenario_014,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction and a personal exemption reduce $88,927.65 of wages to nearly zero. No personal exemption applies, and the $32,200 standard deduction leaves $56,727.65 taxable."
+us,scenario_014,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model improperly included $18,480 of veterans benefits in AGI and invented two children and a $4,000 Child Tax Credit despite the prompt listing no children. It then submitted a negative value that does not follow even from its own stated $4,616.80 tax-after-credit calculation."
+us,scenario_014,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented CTC or ODC eligibility despite no qualifying dependents being listed and incorrectly asserted that the standard deduction reduces taxable income to zero. The $32,200 deduction leaves $56,727.65 taxable, with no listed nonrefundable credit available to erase the $6,311.32 liability."
 us,scenario_014,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract.
 us,scenario_014,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded refundable CTC despite the household having no qualifying children and awarded EITC despite approximately $183,584 of joint income, which exceeds the childless EITC phase-out limit. Those two credits both equal zero, so their asserted $3,067 total has no eligible credit pathway."
 us,scenario_014,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the head's lack of earned income as though Medicaid were tested on the head's income alone, omitting the spouse's wages and other amounts included in household tax-unit MAGI. The resulting MAGI is 4.11 times FPL, above the West Virginia adult limit, and the head has no pregnancy, disability, dependent-child, SSI, or other categorical eligibility pathway."
@@ -811,62 +851,65 @@ us,scenario_014,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
 us,scenario_014,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, violating the required structured-output contract."
 us,scenario_014,payroll_tax,minimax-m3,llm_error,other,False,"The model's own component calculation totals approximately $6,803, but it submitted $6,581. It failed to carry its stated Social Security and Medicare amounts into the final output; the engine components sum to $6,802.96."
 us,scenario_014,payroll_tax,qwen3.8-max,llm_error,other,False,"The model's stated Social Security and Medicare amounts sum to approximately $6,803, not the submitted $8,075.02. It introduced $1,272.02 into the final value without identifying any eligible payroll-tax component."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached essentially the correct $84,927.65 taxable-income base but applied an incorrect approximate 2026 rate schedule and then replaced its own $3,256 calculation with an unsupported $3,374. The applicable brackets produce $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the spouse's $14,717 employer-sponsored insurance premiums from the listed gross wages and treated the resulting $74,211 as the West Virginia base. It then used an average-rate shortcut instead of subtracting the $4,000 personal exemption and applying the statutory brackets to $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"After computing $84,928 of taxable income and about $3,255 under its chosen rates, the model invented an unspecified nonrefundable-credit reduction to $2,877. No applicable nonrefundable credit reduces this household's pre-refundable liability; the correct 2026 brackets yield $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly identified roughly $84,928 of taxable income but mixed approximate rate schedules and then imposed an unsupported further rate-cut adjustment. Applying the actual 2026 West Virginia joint-filer brackets yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model correctly derived approximately $84,928 of taxable income but asserted without calculation that nonrefundable credits reduce the bracket tax to $1,655. No such credit applies, and the 2026 bracket calculation produces $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model repeatedly calculated tax near $3,350–$3,360 using approximate rates, then fabricated cumulative rate reductions and an inapplicable low-income or other nonrefundable credit to force the answer to $1,919. The actual 2026 schedule applied to $84,927.65 yields $3,092.19 with no such credit reduction."
-us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly reached taxable income of about $84,928 but used only a rough range of rates and rounded the liability to $3,200. The exact 2026 joint-filer bracket calculation yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an erroneous $74,211 AGI and invented a $15,400 West Virginia standard deduction. West Virginia instead uses AGI of $88,927.65 and the $4,000 personal exemption, leaving $84,927.65 for the bracket calculation."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used $74,211 as federal AGI, implicitly subtracting the separately listed employer-sponsored premium from gross wages. Those premiums do not create that additional deduction from the wage input; West Virginia taxable income is $84,927.65 after the $4,000 personal exemption."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model supplied no computation and its $4,125 estimate overstates the tax on the applicable $84,927.65 taxable-income base. The West Virginia 2026 joint-filer brackets yield $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an erroneous $74,211 AGI and therefore taxed only $70,211 after exemptions. The correct West Virginia base is $88,927.65 before the $4,000 personal exemption, producing $84,927.65 of taxable income."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly began with $74,211 of federal AGI, reflecting an extra subtraction of employer-sponsored premiums from listed gross wages. The bracket calculation must use $84,927.65 after subtracting only the $4,000 West Virginia personal exemption from AGI."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model taxed $70,211 because it used the premium-reduced $74,211 figure as AGI. PolicyEngine's West Virginia AGI is $88,927.65, so taxable income after two personal exemptions is $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $70,211 of taxable income after starting from a $74,211 AGI that deducts employer-sponsored premiums again. The correct taxable-income base is $84,927.65."
-us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an $8,000 West Virginia standard deduction in addition to the $4,000 personal exemption, then applied the top marginal rate to the entire remainder. West Virginia taxable income is $84,927.65, and each portion must be taxed through the graduated joint-filer brackets."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no usable calculation, and $2,331 understates the result of applying the 2026 West Virginia joint-filer schedule to $84,927.65. That bracket calculation yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented deductions and nonrefundable items that fully eliminate the liability. No listed or applicable nonrefundable provision offsets the tax produced by $84,927.65 of West Virginia taxable income."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly derived about $84,928 of taxable income but used the wrong 2.22%, 2.96%, 3.33%, 4.44%, and 4.82% schedule. The applicable 2026 joint-filer brackets produce $3,092.19, not $3,255.03."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used the correct taxable-income structure but applied an incorrect 2026 West Virginia rate schedule. Applying the actual joint-filer brackets to $84,927.65 produces $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used a projected rather than applicable West Virginia rate schedule. The statutory 2026 joint-filer schedule yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The answer implies the same obsolete approximate brackets that produce about $3,255 on $84,928. The applicable 2026 brackets applied to $84,927.65 yield $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions reduce West Virginia taxable income to zero. AGI of $88,927.65 is reduced by only $4,000 of personal exemptions here, leaving $84,927.65 subject to tax."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly identified approximately $84,928 of taxable income but used a 2024-style schedule as the 2026 baseline. The applicable 2026 rate reductions and joint-filer brackets produce $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model both used the erroneous premium-reduced $74,211 AGI and subtracted the federal standard deduction from the West Virginia base. West Virginia starts from $88,927.65 of AGI and subtracts the $4,000 state personal exemption, not the federal standard deduction."
-us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model identified approximately the correct taxable-income base but used approximate brackets that overstated the liability. Exact application of the 2026 West Virginia joint-filer schedule to $84,927.65 yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $14,717 employer-sponsored premium from gross wages, invented an $8,000 West Virginia standard deduction, and then used a nonexistent 4.5% flat tax. The correct base is $84,927.65 after personal exemptions and is taxed under graduated brackets."
-us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used an obsolete 5.12% top rate and incorrect bracket base tax. The applicable 2026 schedule yields $3,092.19."
-us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated West Virginia's personal exemptions as large enough to erase all wages. The two exemptions total only $4,000, leaving $84,927.65 of taxable income."
-us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the $14,717 employer-sponsored premium from gross wages, used reversed and invented bracket rates, and fabricated a 4% family tax credit. West Virginia taxable income is $84,927.65, and no such credit reduces the resulting $3,092.19 liability."
-us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model wrongly subtracted a federal standard deduction from the West Virginia base and invented nonrefundable credits that eliminate the remaining tax. The state computation instead subtracts the $4,000 personal exemption from $88,927.65 of AGI and taxes $84,927.65 under the 2026 brackets."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived taxable income of about $26,809 and even calculated tax of about $2,969, then submitted $3,103 without a supporting computation. Its final value contradicts its own rate-schedule calculation."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $14,600 standard deduction and $11,600 bracket threshold instead of the 2026 $16,100 deduction and applicable schedule. It then submitted $3,509 despite its own calculation producing $3,094.28."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly reached taxable income near $26,809 and calculated tax near $2,974, but submitted $3,268. The submitted amount has no derivation and contradicts its stated computation."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to reduce wages by the traditional 401(k) contribution, treating the stated $45,000 gross wages as already net. That overstated AGI and taxable income by about $1,042, producing $3,094 instead of applying the schedule to $26,809.38."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $4,510 capital loss even though only $3,000 offsets ordinary income and also omitted the pre-tax 401(k) reduction. It calculated roughly $2,841 from its assumptions but submitted $4,172, so the final answer additionally contradicts its own arithmetic."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted 2025 or speculative 2026 parameters, ultimately using a $15,000 standard deduction and an $11,925 bracket threshold. The required $16,100 standard deduction and 2026 schedule yield $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated approximately $3,011 before credits and $2,876 after its asserted Saver's Credit, then submitted $1,750 based on unnamed adjustments and a nonexistent child credit. No stated computation supports the submitted value."
-us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $2,080 of health-insurance premiums from gross wages and used an obsolete $15,000 standard deduction. The trace permits the pre-tax 401(k), capital-loss, and IRA adjustments but not a separate wage reduction for those premiums."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, replaced the 2026 $16,100 standard deduction with a smaller deduction-plus-exemption regime, and applied a 15% second bracket. The operative 2026 schedule retains the 12% bracket at this taxable income."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied only an approximate AGI and no deduction, bracket, or credit computation supporting $1,541. Applying the $16,100 standard deduction and 2026 schedule to the traced income produces $2,969.13, so its unexplained shortcut omitted substantial regular tax."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by $2,080 of separately listed health-insurance premiums and assumed a post-TCJA-sunset deduction-and-exemption system. Those choices replace the actual $26,809.38 taxable-income derivation and 2026 schedule."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly derived AGI near $42,909 but used a $15,000 standard deduction and an $11,600 bracket threshold. Using the $16,100 standard deduction and applicable 2026 rate schedule lowers the result to $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly treated the $2,080 health-insurance premium as an additional pre-tax wage reduction, lowering AGI to $40,829. The traced income calculation does not deduct that premium and uses a $16,100 standard deduction."
-us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model invoked a standard-deduction-plus-personal-exemption regime that does not govern this 2026 computation. The correct calculation uses the $16,100 standard deduction and the 2026 single-filer schedule, with no personal exemption."
-us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,400 standard deduction and estimated $12,247 bracket threshold. The actual 2026 standard deduction is $16,100, and applying the operative schedule to $26,809.38 yields $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and unspecified nonrefundable credits eliminate tax, but taxable income remains $26,809.38 and no nonrefundable credits apply. Regular tax therefore remains $2,969.13."
-us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated child support as deductible even though personal child-support payments do not reduce federal taxable income. It also supplied no bracket arithmetic or identified credits capable of producing $5,199 from the traced taxable income."
-us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model correctly identified taxable income of about $26,809 and stated the proper 10% and 12% calculation, but those figures equal about $2,969 rather than $3,211.08. The submitted value is an arithmetic or transcription error."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model netted all $1,351 of desired retirement contributions against wages and treated the full $4,510 capital loss as deductible. Only the traditional pre-tax amounts affect income as traced, and the ordinary-income capital-loss deduction is capped at $3,000."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed reversion to pre-TCJA rules, using a personal exemption and a 15% second bracket. The applicable 2026 computation uses a $16,100 standard deduction, no personal exemption, and the 12% bracket at this income."
-us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly applied a projected pre-2018 deduction, personal exemption, and 15% bracket after assuming a TCJA sunset. The operative 2026 law instead produces $26,809.38 of taxable income taxed under the 10% and 12% schedule."
-us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the outstanding mortgage balance and estimated rate, even though no mortgage-interest expense was listed and unlisted numeric inputs are zero. It also overstated deductible medical expenses and SALT, causing an unsupported itemized deduction larger than the $16,100 standard deduction."
-us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. This is a missing-output contract failure rather than a tax computation.
-us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated disability and unspecified credits as sufficient to erase the liability. Disability alone creates neither an additional standard deduction for a 36-year-old nor a credit that eliminates tax here, and $26,809.38 remains taxable with no nonrefundable credits."
-us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model incorrectly included nontaxable workers' compensation and omitted the $49 IRA deduction from its AGI sequence. More decisively, it calculated $3,137.16 under its own assumptions but submitted $2,104, an unsupported final-value substitution."
-us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model treated the one-person household as married filing jointly and invented dependent and child-related credits despite there being no spouse or dependents. It also mishandled gross income and ignored the traced retirement adjustments, so neither its taxable income nor its credits apply."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but applied an incorrect estimated five-rate schedule and then replaced its own $3,256 calculation with an unsupported $3,374. The applicable 2026 joint schedule produces $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the separately listed $14,717 employer-sponsored insurance premium from gross wages and treated the resulting $74,211 as the state tax base. PolicyEngine's wage input produces AGI of $88,927.65, followed by only the $4,000 West Virginia personal exemption before applying the state brackets."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model calculated about $3,255 from its estimated brackets, then arbitrarily reduced the result to $2,877 for unspecified nonrefundable credits despite stating that the family tax credit was unavailable and identifying no other credit. No such reduction applies, and the applicable 2026 schedule yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly identified approximately $84,928 of taxable income but mixed incompatible estimated rate schedules and imposed an unsupported further rate-cut adjustment. Applying the actual 2026 West Virginia joint brackets to $84,927.65 yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model correctly formed approximately $84,928 of taxable income but asserted an unexplained reduction to $1,655 after nonrefundable credits. The household has no applicable nonrefundable credit producing that reduction; the 2026 bracket calculation yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model's own bracket work produced roughly $3,360, but it then invented cumulative rate-trigger reductions and an unidentified low-income or other credit to force the answer down to $1,919. This income does not qualify for the asserted low-income reduction, and the applicable 2026 schedule on $84,927.65 yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly derived approximately $84,928 of taxable income but used only a broad estimated rate range and rounded the liability to $3,200. The exact 2026 West Virginia joint thresholds and rates produce $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an understated $74,211 AGI and then invented a $15,400 West Virginia standard deduction. West Virginia instead subtracts the $4,000 personal exemption from $88,927.65 of AGI, leaving $84,927.65 for the applicable graduated schedule."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used $74,211 as federal AGI, implicitly deducting the separately listed employer-sponsored premium from wages. The wage input generates $88,927.65 of AGI, so after the $4,000 personal exemption the state tax base is $84,927.65, not $70,211."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer supplies no computation beyond saying wage income was taxed, and $4,125 does not result from the applicable graduated schedule. Taxing $84,927.65 after the two personal exemptions yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $74,211 and therefore taxed only $70,211 after personal exemptions. PolicyEngine includes $88,927.65 in AGI and taxes $84,927.65 after the $4,000 exemption."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $74,211 by treating the employer-sponsored premium as an additional wage deduction. The correct state base is $88,927.65 less $4,000 of personal exemptions, or $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $74,211 rather than $88,927.65 as AGI, reducing taxable income to $70,211. Only the $4,000 West Virginia personal exemption is subtracted here, leaving $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model taxed $70,211 because it began with the erroneous $74,211 AGI. The wage input produces $88,927.65 of AGI, and the $4,000 personal exemption leaves $84,927.65 subject to tax."
+us,scenario_014,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an $8,000 West Virginia standard deduction in addition to the $4,000 personal exemption, then applied the top marginal rate to the entire remaining base. There is no such standard deduction in this calculation, and graduated rates apply to $84,927.65."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no usable computation and its $2,331 answer does not follow the applicable West Virginia schedule. AGI of $88,927.65 less the $4,000 personal exemption leaves $84,927.65, on which the exact joint brackets produce $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model asserted that unspecified deductions and nonrefundable items eliminated all liability. The listed facts provide no deduction or credit that offsets the tax on $84,927.65 of West Virginia taxable income, which is $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but applied the wrong 2.22%, 2.96%, 3.33%, 4.44%, and 4.82% schedule. The applicable 2026 joint schedule produces $3,092.19 rather than $3,255.03."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly identified wages less two $2,000 exemptions as the base but used an incorrect 2026 rate calculation. The exact schedule applied to $84,927.65 yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly formed approximately $84,928 of taxable income but applied a projected rather than applicable West Virginia rate schedule. The exact 2026 joint brackets yield $3,092.19, not $3,255.03."
+us,scenario_014,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly described the tax base and absence of credits, so its $3,255 result reflects the wrong bracket rates or thresholds. Applying the applicable 2026 joint schedule to $84,927.65 produces $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions as sufficient to reduce West Virginia taxable income to zero. The calculation subtracts only $4,000 of personal exemptions from $88,927.65 of AGI, leaving $84,927.65 and a positive $3,092.19 liability."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used a 2024-style 2.36% through 5.12% schedule as the 2026 schedule. The applicable 2026 rates and thresholds reduce the liability to $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model invented roughly $20,600 of deductions or exemptions even though this household receives only $4,000 of West Virginia personal exemptions in the traced calculation. The proper taxable income is $84,927.65, not roughly $68,328."
+us,scenario_014,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model both understated AGI as $74,211 and imported the federal standard deduction into the West Virginia tax base. West Virginia starts here from $88,927.65 of AGI and subtracts the $4,000 state personal exemption, not the federal standard deduction."
+us,scenario_014,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly identified taxable income near $84,900 but used approximate brackets that generated $3,422. The exact 2026 West Virginia joint schedule on $84,927.65 yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the separately listed $14,717 employer-sponsored premium from wages, invented an $8,000 West Virginia standard deduction, and then used a nonexistent 4.5% flat tax. The calculation instead taxes $84,927.65 under graduated 2026 rates."
+us,scenario_014,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used a bracket base of $2,183.50 at $60,000 plus a 5.12% top rate. Those are not the applicable 2026 joint schedule parameters, which produce $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the West Virginia personal exemptions as large enough to erase $88,927.65 of AGI. The exemptions total only $4,000, leaving $84,927.65 of taxable income."
+us,scenario_014,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly reached approximately $84,928 of taxable income but used incorrect bracket rates or interval amounts, including a 4.82% top rate. The applicable 2026 joint schedule yields $3,092.19."
+us,scenario_014,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model first understated AGI by subtracting the $14,717 employer premium from wages, then applied a reversed and invented rate schedule and an unsupported 4% family tax credit. The proper base is $84,927.65, and no asserted family-credit reduction applies."
+us,scenario_014,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly imported the federal standard deduction and asserted unspecified nonrefundable credits eliminated the remaining state tax. West Virginia instead taxes $84,927.65 after its $4,000 personal exemption, producing $3,092.19 before refundable credits."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived taxable income of about $26,809 and even computed tax of about $2,969, but then submitted $3,103 without a supporting calculation. Its final value contradicts its own stated bracket arithmetic."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used obsolete $14,600 and $11,600 parameters instead of the 2026 $16,100 standard deduction and applicable rate schedule. It then submitted $3,509 despite its own calculation producing $3,094.28."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model derived taxable income near $26,809 and tax near $2,974, then submitted $3,268 with no computation connecting the two. The submitted amount is inconsistent with its own stated taxable income and brackets."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the traditional 401(k) contribution from the stated gross wages, asserting that the wages were already net even though the prompt reports gross wages and lists the contribution separately. That overstated taxable income by roughly $1,042 and raised the tax."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model incorrectly described the $4,510 capital loss as fully deductible despite the $3,000 annual ordinary-income limit and omitted the traditional 401(k) adjustment. More decisively, it calculated tax of about $2,841 but submitted $4,172 without any supporting step."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted 2025 or speculative projected parameters for the actual 2026 $16,100 standard deduction and rate schedule. Using a $15,000 deduction produced taxable income of $27,909 instead of $26,809.38 and tax of $3,111."
+us,scenario_015,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated liability of about $3,011 and a proposed Saver's Credit reduction to about $2,876, then submitted $1,750 based on unspecified 'additional adjustments.' Disability creates no extra standard deduction at age 36, and there is no child credit because the household has no child."
+us,scenario_015,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model separately deducted $2,080 of employer-sponsored health premiums from gross wages without facts establishing a pre-tax Section 125 payroll exclusion. It also used a $15,000 standard deduction instead of $16,100, producing the wrong taxable-income base."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and replaced the 2026 law with an $8,300 standard deduction, personal exemption, and 15% second bracket. The applicable computation uses the $16,100 standard deduction and 10%/12% schedule on $26,809.38."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no derivation capable of supporting $1,541. On taxable income of $26,809.38, the 2026 single-filer schedule yields $2,969.13, so its answer reflects an unstated and inapplicable reduction."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the separately listed $2,080 health premium from wages and assumed a TCJA-sunset deduction regime. Those steps replaced the traced taxable income of $26,809.38 with an unsupported base and produced $3,509."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction and an $11,600 first-bracket threshold instead of the 2026 $16,100 deduction and applicable schedule. This overstated taxable income by $1,100 and produced $3,117."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model deducted $2,080 of health premiums from gross wages without a stated pre-tax payroll treatment. That erroneous deduction reduced AGI to $40,829 rather than the traced income base, while its $15,700 standard deduction also missed the 2026 amount."
+us,scenario_015,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model invoked a 2026 personal exemption and an unspecified combined deduction regime instead of applying the $16,100 single standard deduction. Its $3,788.85 therefore comes from the wrong deduction and bracket structure."
+us,scenario_015,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,400 standard deduction and estimated $12,247 bracket threshold rather than the governing 2026 parameters. The correct taxable income is $26,809.38 and the schedule produces $2,969.13."
+us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model assumed unspecified nonrefundable credits erased the tax, although this single filer has no qualifying dependent or other available nonrefundable credit. Taxable income remains $26,809.38 and generates $2,969.13 before refundable credits."
+us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated paid child support as a deduction even though personal child-support payments are not deductible. Its $5,199 submission is also incompatible with the lower taxable income it claimed to calculate and does not follow the 2026 rate schedule."
+us,scenario_015,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model correctly identified taxable income of about $26,809 and stated the relevant 10% and 12% brackets, but submitted $3,211.08. Its stated arithmetic equals about $2,969, so the final value is a computation or transcription error."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model netted all $1,351 of desired retirement contributions against wages, even though Roth contributions are not deductible and the traced traditional adjustments are applied separately. It also used the full $4,510 capital loss rather than the $3,000 ordinary-income deduction limit."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed reversion to a pre-TCJA regime with an $8,300 standard deduction, $5,300 personal exemption, and a 15% second bracket. The applicable 2026 computation instead deducts $16,100 and taxes $26,809.38 under the 10%/12% schedule."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used an estimated deduction that left taxable income near $27,159 rather than applying the $16,100 standard deduction to reach $26,809.38. That $350 taxable-income overstatement accounts for most of the difference from $2,969.13."
+us,scenario_015,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed a post-TCJA-sunset combination of standard deduction, personal exemption, and 15% bracket. The governing 2026 calculation uses a $16,100 standard deduction and the 10%/12% schedule, not the reconstructed pre-2018 regime."
+us,scenario_015,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the mortgage balance and an assumed interest rate, even though no mortgage-interest expense or rate was listed and unlisted numeric inputs are zero. It also overstated deductible medical expenses and SALT, causing it to itemize instead of taking the $16,100 standard deduction."
+us,scenario_015,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for the requested output.
+us,scenario_015,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model treated disability and unspecified credits as sufficient to eliminate the liability. At age 36, disability does not create an additional standard deduction, and no available nonrefundable credit reduces the $2,969.13 tax to zero."
+us,scenario_015,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model incorrectly included workers' compensation in taxable income and omitted the $49 IRA adjustment, then calculated tax of $3,137.16. It nevertheless submitted $2,104, a value unsupported by its own derivation."
+us,scenario_015,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model treated the taxpayer as married filing jointly despite the household containing one single head, and invented dependent and child-related credits despite there being no dependent. It also failed to subtract the traditional 401(k) adjustment and combined unrelated tax amounts to reach $4,072.21."
 us,scenario_015,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no parseable value for federal_refundable_credits, violating the required output contract."
 us,scenario_015,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a $1,000 refundable Child Tax Credit despite the household having no qualifying child, and it awarded $1,735.80 of EITC even though this childless adult's approximately $43,450 income exceeds the applicable childless EITC range. Both components are $0, so their sum is $0."
 us,scenario_015,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented qualification through Indiana’s Aged, Blind, and Disabled or working-disabled buy-in pathways from the disability and work facts without applying their actual categorical and financial requirements. The engine assigns category NONE: the head receives no SSI, qualifies through no disability-related pathway, and has MAGI at 2.69 times FPL, so the asserted buy-in eligibility does not exist."
@@ -881,31 +924,33 @@ us,scenario_015,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's stat
 us,scenario_015,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract rather than completing the FICA calculation.
 us,scenario_015,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly calculated $3,442.50 of employee FICA and correctly stated that Indiana adds no mandatory employee state payroll tax, then replaced that total with $3,495 based on nonexistent county-tax adjustments. County income taxes do not belong in this payroll-tax output."
 us,scenario_015,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model improperly deducted the $1,042 traditional 401(k) contribution from the FICA wage base; traditional 401(k) deferrals remain subject to Social Security and Medicare taxes. Its submitted $3,425.67 also contradicts its own stated component total of $3,362.79, while applying both rates to the full $45,000 yields $3,442.50."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived the correct $41,909 taxable base and 2.95% rate, which produce $1,236.33, but then replaced the computed result with $1,234 without any valid rounding operation."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete 3.15% Indiana rate instead of 2.95% and treated workers’ compensation and the full capital loss incorrectly in its initial income calculation. It then inserted unsupported disability adjustments to reach $1,255."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model invented an additional $500 disability exemption and used 3.05% rather than Indiana’s 2026 rate of 2.95%. Its submitted $1,352 also contradicts every intermediate tax amount in its reasoning."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $1,042 traditional 401(k) contribution when computing AGI, leaving its base $1,042 too high. It then submitted $1,392 even though neither 2.95% nor 3.05% applied to its stated base produces that amount."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an incorrect AGI of $42,441 and an incorrect 2.85% Indiana rate. The trace requires $42,909.38 of AGI, a $1,000 exemption, and a 2.95% rate."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted a $1,000 disability exemption and $2,080 of health-insurance premiums from Indiana AGI. Only the $1,000 base exemption applies here, leaving $41,909.38 taxable rather than $38,829."
-us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model improperly added a $1,000 disability exemption and used 3.05% instead of 2.95%. Its submitted $1,350 also does not follow from its stated $40,909 base and rate."
-us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an unexplained federal AGI of $40,829 instead of $42,909.38 and used 3.05% instead of 2.95%. The correct taxable base after the exemption is $41,909.38."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $3,000 of child support and an additional $1,500 disability exemption. Child support is not deductible and the trace applies only the $1,000 base exemption; it also used 3.0% rather than 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 2.79% Indiana rate instead of the 2026 rate of 2.95% and supplied no valid taxable-income derivation. Applying 2.95% to $41,909.38 yields $1,236.33."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $40,829 instead of $42,909.38. Although it used the correct 2.95% rate and $1,000 exemption, the wrong starting income reduced its result."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified the $41,909 taxable base but applied 3.05% instead of Indiana’s 2026 rate of 2.95%. The correct rate produces $1,236.33."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $40,829 and applied 3.0% rather than 2.95%. The correct calculation taxes $41,909.38 after the $1,000 exemption."
-us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $41,909 but used a 3.0% rate instead of 2.95%. That rate error accounts for its excess tax."
-us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted the full $4,510 capital loss rather than applying the $3,000 federal capital-loss limit, understating AGI by $1,510. It also used 3.0% rather than the 2026 Indiana rate of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies unsupported state standard deductions or exemptions that reduced the traced $41,909.38 taxable base. Indiana’s calculation here subtracts only the $1,000 base exemption from $42,909.38 and then applies 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model zeroed the liability through unspecified deductions and credits even though the requested measure is before refundable credits. Taxing $41,909.38 at 2.95% leaves $1,236.33 of state tax."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly derived the taxable base but used a nonexistent 3.025% rate instead of Indiana’s 2026 rate of 2.95%. The correct rate yields $1,236.33."
-us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted two separate $1,000 amounts from federal AGI, double-counting Indiana’s sole base exemption. Only one $1,000 exemption applies, so taxable income is $41,909.38 rather than $40,909."
-us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used a vague wage-based estimate and invoked a standard deduction that is not part of this Indiana computation. The required base is federal AGI of $42,909.38 less the $1,000 exemption, taxed at 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model correctly identified the taxable base but applied 2.85% instead of Indiana’s 2026 rate of 2.95%. Applying the correct rate produces $1,236.33."
-us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
-us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used 3.05% rather than Indiana’s 2026 rate of 2.95% and did not state a reproducible taxable-income base. The traced base is $41,909.38, producing $1,236.33."
-us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated workers’ compensation as taxable and deducted the full $4,510 capital loss instead of the $3,000 federal limit, then failed to subtract Indiana’s $1,000 base exemption. It also applied 3.05% instead of 2.95%."
-us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly used estimated federal taxable income after a federal deduction as Indiana’s tax base. Indiana starts from federal AGI of $42,909.38, subtracts its own $1,000 exemption, and taxes the result at 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived the correct $41,909 taxable base and 2.95% rate, which produce $1,236.33, then discarded that computation and submitted $1,234 as unsupported “minor rounding.”"
+us,scenario_015,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included nontaxable workers’ compensation in gross income, used the full $4,510 capital loss instead of the $3,000 federal limit, and invoked unsupported child-support and disability adjustments. It also used 3.15% instead of Indiana’s 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented an additional $500 disability exemption even though the trace allows only the $1,000 base exemption. It then submitted $1,352 despite its own stated calculations producing neither that amount nor the correct 2.95%-rate liability."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $1,042 traditional 401(k) contribution when deriving federal AGI, overstating the Indiana tax base. It also replaced the applicable 2.95% rate calculation with an unsupported $1,392 estimate."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model understated federal AGI and applied a nonexistent 2.85% Indiana rate. The correct base is $41,909.38 after the $1,000 exemption and the applicable 2026 rate is 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted a $1,000 disability exemption and $2,080 of health-insurance premiums from Indiana AGI. Only the $1,000 base exemption applies, leaving $41,909.38 taxable at 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented a $1,000 disability exemption and used 3.05% rather than 2.95%. It then submitted $1,350 even though its own erroneous calculation yielded about $1,248."
+us,scenario_015,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model started from an unexplained federal AGI of $40,829 instead of $42,909.38 and applied 3.05% rather than 2.95%. Indiana taxable income is $41,909.38 after the sole $1,000 exemption."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $3,000 child-support deduction and a $1,500 disability exemption. Child support is not deductible here, and only the $1,000 base exemption reduces the $42,909.38 Indiana AGI; it also used 3.0% instead of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted a 2.79% Indiana rate, but the applicable 2026 rate is 2.95%. Its $1,256 answer also does not follow from 2.79% applied to any stated taxable-income base."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $40,829 rather than $42,909.38. Subtracting the $1,000 exemption from the correct AGI yields $41,909.38, not $39,829."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly derived taxable income of about $41,909 but applied 3.05%, an earlier Indiana rate, instead of the 2026 rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $40,829 and used 3.0% instead of 2.95%. The correct taxable base after Indiana’s $1,000 exemption is $41,909.38."
+us,scenario_015,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly identified the approximately $41,909 taxable base but applied a 3.0% rate. Indiana’s 2026 rate is 2.95%, producing $1,236.33."
+us,scenario_015,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model deducted the full $4,510 capital loss and omitted the traditional 401(k) and IRA treatment that produces federal AGI of $42,909.38. It also applied 3.0% instead of the 2026 Indiana rate of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies unsupported standard deductions or exemptions that reduced Indiana’s tax base too far. Indiana uses $42,909.38 of AGI less only the $1,000 base exemption, yielding $1,236.33 at 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed deductions and credits eliminated the liability. The $42,909.38 Indiana AGI is reduced only by the $1,000 base exemption, leaving a positive $41,909.38 tax base."
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model correctly derived the Indiana taxable base but applied a nonexistent 3.025% rate. The applicable 2026 flat rate is 2.95%.
+us,scenario_015,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model subtracted both an unexplained $1,000 deduction and the $1,000 personal exemption, double-reducing Indiana AGI. Only one $1,000 base exemption applies, leaving $41,909.38 taxable."
+us,scenario_015,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used wages as its starting point and referred generically to a standard deduction rather than deriving Indiana AGI from federal AGI. The required base is $42,909.38 less the $1,000 Indiana exemption, taxed at 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly derived taxable income of about $41,909 but applied 3.0%. The 2026 Indiana rate is 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model correctly identified the taxable-income base but used a nonexistent 2.85% rate. Applying Indiana’s 2026 rate of 2.95% to $41,909.38 yields $1,236.33."
+us,scenario_015,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for the requested variable, so the required numeric value and supporting explanation were missing."
+us,scenario_015,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used 3.05% rather than Indiana’s 2026 rate of 2.95% and did not state a reproducible taxable-income base. The correct calculation taxes $41,909.38 at 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model correctly derived the $41,909 taxable base but applied 2.97%. Indiana’s applicable 2026 rate is 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated workers’ compensation as taxable and deducted the full $4,510 capital loss rather than limiting it to $3,000, producing the wrong federal AGI. It also failed to subtract Indiana’s $1,000 base exemption and used 3.05% instead of 2.95%."
+us,scenario_015,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model substituted federal taxable income after the federal standard deduction for Indiana’s federal-AGI starting point. Indiana instead uses $42,909.38 of AGI less its own $1,000 exemption, and the applicable rate is 2.95%, not 3.05%."
 us,scenario_015,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_016,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model correctly calculated that itemizing the medical expenses reduces taxable income and tax to zero, but then discarded that result and submitted a standard-deduction computation. Its $1,523 answer contradicts its own conclusion that the taxpayer optimally itemizes."
 us,scenario_016,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model ignored the itemized medical-expense deduction that eliminates taxable income. It also invented mortgage interest from the mortgage balance and a Saver's Credit without any listed interest payment or retirement contribution, neither of which can replace the required computation from the supplied inputs."
@@ -920,32 +965,33 @@ us,scenario_016,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibi
 us,scenario_016,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model treated payroll tax as requiring a separately listed payroll-tax input and ignored that employee FICA is automatically imposed on the listed $34,638 of wages. It omitted both the 6.2% employee Social Security tax and the 1.45% employee Medicare tax."
 us,scenario_016,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated both applicable FICA components and even obtained an intermediate total near $2,648.40, but then submitted $2,157.70 without adding Social Security and Medicare correctly. The traced components are $2,147.55 and $502.25, which sum to $2,649.80."
 us,scenario_016,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly identified every applicable payroll-tax component and computed a total near $2,650, then replaced that result with an unsupported $2,674 'minor adjustment.' No adjustment or rounding rule adds $24.20; the component amounts sum directly to $2,649.80."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,625 of auto-loan interest even though the facts do not establish a qualified vehicle loan for the temporary deduction. Taxable income remains $45,492.09 after the $16,100 standard deduction, yielding $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $14,600 standard deduction and incorrect 12% and 22% bracket thresholds instead of the 2026 $16,100 deduction and applicable brackets. Its submitted $4,903 also contradicts its own stated calculation of $5,716."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived taxable income of about $45,492 and tax of about $5,211, then replaced that result with $6,754 without any supporting computation. Its stated bracket arithmetic directly yields the reference amount rather than its submitted value."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified $45,492 of taxable income and computed $1,240 plus approximately $3,971, but then reported $6,033. Those components sum to approximately $5,211, so the error is an unsupported final aggregation."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used a $15,000 standard deduction instead of $16,100 and then subtracted a $200 Saver's Credit based on an incorrect 2026 eligibility schedule. No nonrefundable credit reduces the $5,211.05 regular tax in this case."
-us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"After deriving essentially the correct taxable income and regular tax, the model subtracted an unspecified retirement-savings credit to reach $4,913. No Saver's Credit or other nonrefundable credit applies, so the bracket tax remains $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA rules expired for 2026, restoring a personal exemption, a small standard deduction, and 15% brackets. It also treated the listed $9,000 health premium as a pre-tax wage exclusion without such an input; the applicable standard deduction is $16,100 and taxable income is $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset regime with a personal exemption and 15% and 25% rates. The 2026 computation instead uses the $16,100 single standard deduction and 10% and 12% brackets on $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 standard deduction, omitted the $50 of taxable interest and $288.51 IRA deduction, and treated the resulting taxable income as though tax were a flat 10%. The correct taxable income is $45,492.09 and must be taxed progressively."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed pre-TCJA rules, combining an estimated standard deduction and personal exemption and applying reverted brackets. The applicable deduction is the $16,100 standard deduction with no personal exemption, producing $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly restored a personal exemption and 15% bracket under a supposed TCJA sunset. It also subtracted $9,000 of health premiums from wages without a stated pre-tax exclusion; AGI is $61,592.09 and taxable income is $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and incorrect bracket threshold instead of the 2026 $16,100 deduction and applicable single-filer brackets. This overstated taxable income by $400 and produced $5,261.54 rather than $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-sunset regime with a personal exemption and 15% bracket. The 2026 rules use a $16,100 standard deduction, no personal exemption, and produce $5,211.05 on $45,492.09 of taxable income."
-us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model estimated the standard deduction as $15,400 and the bottom-bracket ceiling as $12,260. The correct $16,100 deduction and 2026 bracket parameters reduce taxable income to $45,492.09 and tax to $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented one qualifying child and subtracted a nonrefundable Child Tax Credit even though the household contains only the 53-year-old head. With no qualifying dependent and no applicable nonrefundable credit, tax remains $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model assumed unspecified nonrefundable credits eliminated the entire liability despite identifying no qualifying credit. The stated facts produce $45,492.09 of taxable income and $5,211.05 of tax before refundable credits."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invented a $6,760 qualified-overtime deduction by inferring overtime from hours and hourly pay. Gross wages already include all overtime, and no deductible qualified-overtime amount is listed, so that additional deduction is zero."
-us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly derived AGI and the $16,100 standard deduction but used bracket parameters that produced $5,220.54. Applying the exact 2026 single-filer thresholds and rates to $45,492.09 yields $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the progressive-tax calculation to an unsupported $5,200 estimate instead of applying the exact 2026 bracket parameters. Tax on $45,492.09 of taxable income is $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted or invented medical deductions to reach $18,500, added an unlisted SALT deduction, and subtracted a nonexistent 2026 personal exemption. The standard deduction is $16,100, and the reverted 15% bracket regime it used does not apply."
-us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an estimated $15,375 standard deduction and estimated brackets instead of the exact 2026 parameters. The $16,100 standard deduction leaves $45,492.09 taxable and produces $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model treated roughly $15,381 of medical costs and an unlisted state-tax amount as itemized deductions. The listed expenses do not produce itemized deductions exceeding the $16,100 standard deduction, which leaves taxable income of $45,492.09."
-us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not be parsed or substantively evaluated."
-us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model set the liability to zero without applying the income, deduction, and bracket computation. The facts yield AGI of $61,592.09, taxable income of $45,492.09, and tax of $5,211.05."
-us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model failed to deduct the $288.51 traditional IRA contribution and used an estimated $15,750 standard deduction with incorrect bracket thresholds. The correct AGI is $61,592.09 and the $16,100 standard deduction leaves $45,492.09 taxable."
-us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to about $32,942 by deducting retirement contributions again after they had already reduced wages or AGI, then invented a $303 Saver's Credit. Taxable income is $45,492.09 and no nonrefundable credit reduces the resulting $5,211.05 tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $1,625 of auto-loan interest from taxable income. The trace allows only the $16,100 standard deduction after AGI, leaving $45,492.09 taxable and producing $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction and incorrect 2026 bracket thresholds instead of the $16,100 deduction and applicable 10% and 12% brackets. Its submitted $4,903 also contradicts its own stated calculation of $5,716."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived approximately $5,211 from $45,492 of taxable income, then replaced that result with $6,754 without any computation supporting the adjustment. The stated bracket arithmetic directly yields the reference amount, not the submitted value."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified the taxable income and computed $1,240 plus approximately $3,971, but added those components incorrectly. They total about $5,211, not $6,033."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used an outdated $15,000 standard deduction and then awarded a $200 Saver's Credit based on incorrect 2026 eligibility thresholds. The applicable $16,100 standard deduction and no credit yield $45,492.09 of taxable income and $5,211.05 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model's bracket calculation already produced about $5,216, but it then subtracted an unspecified Saver's Credit despite describing that credit as phased out. No nonrefundable credit applies, and the exact 2026 brackets produce $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model treated the $9,000 health premium as a pre-tax AGI reduction and assumed that TCJA rules expired, restoring a personal exemption and 15% bracket. The trace instead gives AGI of $61,592.09 and applies the $16,100 standard deduction under the operative 2026 brackets."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset regime with a small standard deduction, a personal exemption, and 15% and 25% rates. The operative 2026 calculation uses a $16,100 standard deduction and 10% and 12% brackets on $45,492.09."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 standard deduction, omitted the $289 deductible traditional IRA contribution and $50 of taxable interest, and reduced the resulting taxable income to tax using an unsupported shortcut. The correct taxable income is $45,492.09 and its bracket tax is $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed TCJA expiration and combined an estimated standard deduction with a restored personal exemption totaling only $13,790. The operative rule supplies a $16,100 standard deduction and no personal exemption, with 10% and 12% rates producing $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model improperly deducted $9,000 of health premiums from AGI and applied a sunset regime with a personal exemption and 15% marginal rate. AGI is $61,592.09, taxable income is $45,492.09 after the $16,100 standard deduction, and the applicable marginal rate is 12%."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an estimated $15,700 standard deduction and incorrect bracket cutoff rather than the exact 2026 parameters. Using the $16,100 deduction and applicable bracket thresholds reduces its taxable income to $45,492.09 and tax to $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied an inapplicable TCJA-sunset calculation with a personal exemption and 15% rate. The operative 2026 rules use the $16,100 standard deduction without a personal exemption and tax $45,492.09 at the 10% and 12% brackets."
+us,scenario_018,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted an estimated $15,400 standard deduction and estimated bracket threshold for the exact 2026 parameters. The $16,100 deduction leaves $45,492.09 taxable, whose bracket tax is $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a qualifying child and subtracted a nonrefundable Child Tax Credit even though the household contains only the 53-year-old head. With no dependent and no applicable credit, the tax remains $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model zeroed out the liability through unspecified nonrefundable credits that the household does not qualify for. The listed deductions leave $45,492.09 taxable, and the regular bracket tax remains $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model deducted $6,760 as qualified overtime without deriving qualified overtime compensation from the listed annual wages, hourly rate, and hours. The trace includes no overtime deduction, so taxable income remains $45,492.09 and tax is $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly obtained AGI, the $16,100 standard deduction, and no credits, but applied the 2026 tax brackets incorrectly. The bracket calculation on $45,492.09 equals $5,211.05, not $5,220.54."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded an approximate tax estimate instead of applying the exact 2026 brackets to $45,492.09 of taxable income. That exact computation yields $5,211.05 rather than $5,200."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model counted unsupported medical and SALT itemized deductions, restored a personal exemption, and used reverted 10% and 15% rates. The correct path takes the $16,100 standard deduction, no personal exemption, and the operative 10% and 12% brackets."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a $15,400 standard deduction and an incorrect 10% bracket endpoint. The exact $16,100 deduction and 2026 thresholds produce taxable income of $45,492.09 and tax of $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted a $15,375 estimated standard deduction and unspecified estimated brackets. The exact 2026 standard deduction is $16,100, and the bracket tax on the resulting $45,492.09 is $5,211.05."
+us,scenario_018,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model constructed itemized deductions from unsupported medical and state-tax amounts and treated them as exceeding the standard deduction. The calculation instead uses the $16,100 standard deduction, leaving $45,492.09 taxable and $5,211.05 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required output contract before any substantive tax computation could be evaluated.
+us,scenario_018,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The unexplained zero treats the regular income-tax liability as fully eliminated despite the absence of applicable nonrefundable credits. The listed income and deductions leave $45,492.09 taxable and generate $5,211.05 of tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $289 traditional IRA deduction and used an estimated $15,750 standard deduction instead of $16,100. Those errors raised taxable income above the correct $45,492.09 and produced an overstated tax."
+us,scenario_018,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to about $32,942 by double-counting or otherwise treating retirement contributions as deductions beyond their proper AGI treatment, then subtracted an inapplicable $303 Saver's Credit. Retirement deductions and the standard deduction instead leave $45,492.09 taxable, with no nonrefundable credit reducing the $5,211.05 tax."
 us,scenario_018,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_018,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly computed $4,216.31 of Social Security tax and $986.07 of Medicare tax, which sum exactly to $5,202.38, then invented a $49.41 “rounding adjustment.” No such adjustment applies."
 us,scenario_018,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the listed $9,000 health-insurance premiums from FICA wages as though they were documented pre-tax employer-plan salary reductions. Applying 7.65% to the full $68,005 wage base yields $5,202.38."
@@ -955,67 +1001,71 @@ us,scenario_018,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated
 us,scenario_018,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so it failed the required output contract."
 us,scenario_018,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omits federal employee FICA on $68,005 of wages. The correct computation includes $4,216.31 of Social Security tax and $986.07 of Medicare tax, totaling $5,202.38."
 us,scenario_018,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own correct standard-deduction calculation produced about $1,146, but it submitted $1,503 with no supporting computation. Its discussion of a car-loan-interest adjustment was also irrelevant to the submitted figure and did not replace Arizona’s $15,750 standard deduction."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete progressive Arizona rate schedule instead of the 2.5% flat rate and used a $14,600 deduction instead of $15,750. It then asserted $2,175 without deriving that amount from either its bracket calculation or its medical-expense discussion."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model estimated the Arizona single standard deduction as $15,000 instead of $15,750. That left taxable income $750 too high and produced $1,165 rather than 2.5% of $45,842.09."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model used an incorrect $14,600 standard deduction, whose stated calculation yielded about $1,175, and then submitted $1,471 without any corresponding tax computation. The correct $15,750 deduction leaves $45,842.09 taxable at 2.5%."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model’s stated $16,100 deduction calculation yielded about $1,137, but it submitted $1,481. Both the deduction and the final transcribed value were wrong; Arizona uses $15,750 here."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted a projected $14,600 standard deduction for Arizona’s 2026 amount of $15,750. Applying 2.5% after the correct deduction yields $1,146.05, not $1,175."
-us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s stated AGI and $14,600 deduction calculation produced about $1,175, but it submitted $1,450 after an unsupported premium adjustment. Health-premium considerations do not justify that submitted amount, and the applicable standard deduction is $15,750."
-us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $52,592 and used a nonexistent $7,700 standard deduction. The trace uses $61,592.09 of AGI and the $15,750 Arizona standard deduction, leaving $45,842.09 taxable."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly used the 2.5% rate but used a projected $14,600 deduction instead of Arizona’s $15,750 deduction. The extra $1,150 deduction lowers tax by $28.75 to $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied only an unsupported estimate and did not calculate Arizona taxable income. Subtracting the $15,750 standard deduction from $61,592.09 and applying 2.5% yields $1,146.05, not $1,400."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an estimated $13,790 standard deduction instead of $15,750. This overstated taxable income and therefore overstated the flat-rate tax."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Arizona AGI by $9,000, using $52,592 rather than $61,592.09. With the correct AGI and $15,750 standard deduction, taxable income is $45,842.09."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction, $50 below the applicable $15,750 amount. That made taxable income $50 too high and tax $1.25 too high."
-us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model’s $46,092 taxable-income estimate implies a $15,500 deduction rather than $15,750. The correct taxable income is $45,842.09, producing $1,146.05 at 2.5%."
-us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model misplaced the decimal when converting its own multiplication into the submitted value: $46,192 at 2.5% is about $1,154.80, not $11,548. It also used $15,400 rather than the applicable $15,750 deduction."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model submitted $1,941 without identifying any taxable-income or rate calculation that produces it. The correct computation applies 2.5% to $61,592.09 less the $15,750 standard deduction, yielding $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model zeroed out the liability despite positive taxable income and no offsetting nonrefundable credits. Arizona taxes $45,842.09 at 2.5%, producing $1,146.05 before refundable credits."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $18,500 of medical and premium expenses as the operative deduction. Arizona instead selects the $15,750 standard deduction because the trace’s allowable itemized deductions total only $10,786.71."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a $16,100 standard deduction instead of $15,750. This understated taxable income by $350 and tax by $8.75."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used a $16,100 single standard deduction rather than Arizona’s $15,750 amount. The correct taxable base is $45,842.09, not $45,492."
-us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model’s $45,492 taxable-income estimate embeds a $16,100 deduction instead of $15,750. Correcting the deduction raises the flat-rate liability to $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model treated Arizona tax as progressive and applied an unexplained average rate to roughly $61,000. Arizona applies a 2.5% flat rate to $45,842.09 after the $15,750 deduction."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a projected $15,450 standard deduction instead of $15,750. That overstated taxable income by $300 and tax by $7.50."
-us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $15,375 standard deduction instead of $15,750. The $375 difference overstated tax by $9.375 before rounding."
-us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model combined an incorrect $16,100 standard deduction with an additional unsupported $2,300 exemption. The trace permits the single $15,750 standard deduction, leaving $45,842.09 taxable."
-us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required numeric result was missing."
-us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of $15,750. This overstated taxable income by $750 and tax by $18.75."
-us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model set the liability to zero without applying Arizona’s tax calculation. The $15,750 standard deduction leaves $45,842.09 taxable, and the 2.5% rate produces $1,146.05."
-us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the listed $68,005 gross wages as already reduced by the desired traditional 401(k) contribution. The benchmark inputs require subtracting the $6,174 traditional 401(k) contribution and $289 IRA deduction, producing $61,592.09 of AGI before the $15,750 Arizona deduction."
-us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an incorrect $14,508 deduction and an unstated rate structure that produced $832 from about $45,486 of taxable income. Arizona instead uses the $15,750 deduction and applies its 2.5% flat rate to $45,842.09."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own correct calculation—$61,592 AGI minus the $15,750 standard deduction, taxed at 2.5%—produces about $1,146.05, but it submitted $1,503. It also introduced an inapplicable auto-loan-interest adjustment that was not used in the submitted figure."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete progressive Arizona rate schedule instead of the 2.5% flat rate and used $14,600 rather than the $15,750 standard deduction. It then submitted $2,175 despite its own bracket calculation producing $1,304.40."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction at $15,000 instead of applying the 2026 Arizona amount of $15,750. The correct taxable income is $45,842.09, not $46,592."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model used the wrong $14,600 standard deduction, which would yield about $1,175, and then submitted an unrelated $1,471. The submitted value does not follow from its stated AGI, deduction, or 2.5% rate."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model used an incorrect $16,100 standard deduction and correctly observed that its resulting tax was about $1,137, but submitted $1,481. Its final value is disconnected from every computation in its explanation."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model projected a $14,600 Arizona standard deduction instead of using $15,750. This overstated taxable income by $1,150 and tax by about $28.75."
+us,scenario_018,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model used the wrong $14,600 standard deduction and computed approximately $1,175, then replaced that result with an unsupported $1,450 adjustment. No health-premium adjustment turns its stated calculation into the submitted amount."
+us,scenario_018,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $52,592 and used an incorrect $7,700 standard deduction. The correct inputs are $61,592.09 of AGI and a $15,750 standard deduction, yielding $45,842.09 of taxable income."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a projected $14,600 standard deduction rather than Arizona’s $15,750 amount. Its AGI and 2.5% rate were otherwise aligned with the required computation."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,400 does not result from subtracting the $15,750 standard deduction from $61,592.09 and applying the 2.5% rate. The correct deduction step leaves $45,842.09 taxable and produces $1,146.05."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an estimated $13,790 standard deduction instead of $15,750. That overstated Arizona taxable income and therefore the tax."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Arizona AGI by $9,000, using $52,592 instead of $61,592.09, and also used a $14,600 deduction instead of $15,750. Those errors reduced taxable income to $37,992 rather than $45,842.09."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $15,700 standard deduction, $50 below the applicable $15,750 amount. That left taxable income $50 too high and tax $1.25 too high."
+us,scenario_018,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $46,092 rather than $45,842.09. This reflects a standard deduction about $250 below the applicable $15,750 amount."
+us,scenario_018,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model used the wrong $15,400 standard deduction and then misplaced a decimal in multiplying $46,192 by 2.5%. That multiplication is about $1,154.80, not $11,548."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $1,941 does not follow from the Arizona tax base: $61,592.09 AGI minus the $15,750 standard deduction equals $45,842.09. Taxing that amount at 2.5% yields $1,146.05."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model zeroed out Arizona income tax despite positive taxable income and no nonrefundable credit eliminating the liability. The 2.5% rate applied to $45,842.09 produces $1,146.05."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly used $18,500 of medical and premium expenses as a deduction. Arizona itemized deductions total only $10,786.71 after the applicable medical-expense rules, so the larger $15,750 standard deduction must be selected."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used a $16,100 standard deduction instead of $15,750. This understated taxable income by $350 and tax by $8.75."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted an incorrect $16,100 standard deduction from AGI. The applicable $15,750 deduction leaves $45,842.09 taxable, not $45,492."
+us,scenario_018,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model’s $45,492 taxable-income estimate embeds a $16,100 deduction rather than the applicable $15,750 standard deduction. It therefore understated tax by $8.75."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model treated Arizona tax as progressive and effectively taxed roughly $61,000 rather than the $45,842.09 taxable-income base. Arizona applies a flat 2.5% rate after the $15,750 standard deduction."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model projected a $15,450 standard deduction instead of using $15,750. This overstated taxable income by about $300 and tax by about $7.50."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,400 standard deduction rather than $15,750. That overstated taxable income by about $350 and tax by about $8.75."
+us,scenario_018,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $15,375 standard deduction instead of $15,750. This left taxable income about $375 too high and overstated tax by about $9.38."
+us,scenario_018,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted both an incorrect $16,100 standard deduction and an unsupported $2,300 exemption. The correct computation subtracts only the selected $15,750 standard deduction from $61,592.09."
+us,scenario_018,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured answer was missing."
+us,scenario_018,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction instead of $15,750. This overstated taxable income by $750 and tax by $18.75."
+us,scenario_018,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model returned zero even though $61,592.09 of AGI minus the $15,750 deduction leaves $45,842.09 subject to Arizona’s 2.5% tax. No stated nonrefundable credit offsets the resulting $1,146.05 liability."
+us,scenario_018,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used a $16,100 standard deduction rather than $15,750. The family-tax-credit observation does not cure the resulting $350 understatement of taxable income."
+us,scenario_018,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $6,174 traditional 401(k) contribution and the $289 deductible traditional IRA contribution when deriving AGI. The benchmark’s gross-wage input is before those modeled contributions, so Arizona AGI is $61,592.09, not $68,055; it also used $14,600 rather than the $15,750 standard deduction."
+us,scenario_018,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an incorrect Arizona rate structure: $832 on roughly $45,486 is not the result of the 2.5% flat rate. It also used the wrong $14,508 deduction; the correct base is $45,842.09 after the $15,750 standard deduction."
 us,scenario_018,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model limited itemized deductions to the listed $13,834 real-estate tax and therefore chose the $16,100 standard deduction. It missed the full $17,090.34 state and local tax deduction, so taxable income should be $283,059.84 rather than $284,050."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $138,375 of unreimbursed employee business expenses from AGI and then produced a final number inconsistent with its own stated $146,775 taxable income. Those expenses do not enter the reference computation; AGI is $300,150.19 and taxable income is $283,059.84 after itemizing SALT."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model double-counted the $42,857 FLSA overtime premium even though gross wages are explicitly the annual total including overtime. It also capped SALT at $10,000 and chose the standard deduction instead of applying the $17,090.34 itemized SALT deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model denied the specified alimony deduction and invented $23,855 of mortgage interest from the mortgage balance despite the instruction that unlisted expenses are zero. The computation instead deducts $5,708.82 above the line and itemizes $17,090.34 of SALT, with no imputed mortgage interest."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model replaced the deduction calculation with an unsupported estimate of $30,000–$40,000 and never computed taxable income from the actual itemized amount. The applicable itemized SALT deduction is $17,090.34, producing taxable income of $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model rejected the $5,708.82 alimony adjustment and limited SALT to $10,000, causing it to choose an estimated standard deduction. The correct computation reduces AGI to $300,150.19 and itemizes $17,090.34, leaving $283,059.84 taxable."
-us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 and used a $15,000 standard deduction instead of the $17,090.34 itemized deduction. It also added the $900 Additional Medicare Tax to this income-tax output even though that amount belongs in employee payroll tax, then abandoned its own $70,467 calculation for an unsupported $76,500."
-us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted employer-sponsored insurance premiums from wages, restored miscellaneous employee-expense deductions and a personal exemption, and used reverted pre-TCJA brackets. The computation instead starts from the stated $305,859 gross income, deducts alimony, itemizes $17,090.34 of SALT, and applies the operative 2026 brackets."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the separately listed insurance premiums and treated unreimbursed employee expenses and a personal exemption as regular-tax deductions, then substituted AMT for the regular-tax result. The correct taxable-income pathway uses the stated wages, the alimony adjustment, and $17,090.34 of itemized SALT, yielding $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model left AGI at $305,859, omitted the $5,708.82 alimony deduction, and limited SALT to $10,000. Applying the alimony adjustment and the full $17,090.34 SALT deduction produces taxable income of $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model assumed a TCJA sunset, deducted insurance premiums from stated gross wages, restored the unreimbursed-employee-expense deduction, and relied on AMT. Those adjustments are absent from the operative computation, which itemizes $17,090.34 of SALT after the alimony deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced stated wages by employer insurance premiums and deducted the large unreimbursed employee expenses, driving AGI and regular taxable income far below the traced amounts. AGI is $300,150.19 after alimony, and taxable income is $283,059.84 after itemized SALT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted insurance premiums from stated gross wages and restored miscellaneous employee-expense deductions and personal exemptions. The operative calculation allows neither shortcut and instead deducts $17,090.34 of SALT from $300,150.19 AGI."
-us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model treated 2026 as a reversion to pre-TCJA miscellaneous deductions and personal exemptions and also reduced gross wages by insurance premiums. The correct computation uses the stated wages, deducts alimony, and itemizes only the traced $17,090.34 SALT amount."
-us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model restored the deduction for unreimbursed employee business expenses, a personal exemption, and pre-TCJA tax brackets. The applicable computation does not deduct those employee expenses or an exemption; taxable income is $283,059.84 after the $17,090.34 SALT deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The zero answer treats the mortgage balance and unreimbursed employee expenses as deductions large enough to eliminate taxable income. No mortgage interest was listed, the employee expenses do not reduce this computation, and taxable income remains $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $31,472 answer reflects itemized deductions far beyond the traced $17,090.34, consistent with deducting the $138,375 employee expenses under reverted rules. Those expenses do not reduce taxable income, which is $283,059.84 before applying the 2026 rates."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model chose the $16,100 standard deduction after considering only the listed property tax. It missed the full $17,090.34 SALT itemized deduction, overstating taxable income by $990.16."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated employer-sponsored insurance premiums as an additional adjustment to the stated gross wages and used the standard deduction. Gross income remains $305,859, AGI becomes $300,150.19 after alimony, and the filer itemizes $17,090.34 of SALT."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction plus a separate $196 nonitemizer charitable deduction. The applicable deduction is instead the larger $17,090.34 itemized SALT amount, producing $283,059.84 of taxable income."
-us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction and omitted the traced alimony and itemized-deduction pathway from its explanation. The correct computation deducts $5,708.82 from gross income and then itemizes $17,090.34 of SALT."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an approximate $306,000 AGI and standard deduction rather than the traced inputs. AGI is $300,150.19 after alimony, and the filer itemizes $17,090.34 to reach $283,059.84 taxable income."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model restored the miscellaneous itemized deduction for employee expenses, a personal exemption, and pre-TCJA brackets. Those provisions do not enter the computation; only the traced $17,090.34 itemized SALT deduction reduces AGI below the line."
-us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model reduced stated wages by insurance premiums and applied pre-TCJA employee-expense deductions and brackets. The reference pathway retains the full $300,000 wages, deducts alimony, and itemizes $17,090.34 of SALT."
-us,scenario_020,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced stated wages by $8,389 of employer-sponsored insurance premiums and treated the $196 charity as an above-the-line deduction. Gross wages already supply the tax input; the actual adjustments are the $5,708.82 alimony deduction and $17,090.34 itemized SALT deduction."
-us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required submission contract."
-us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the $16,100 standard deduction and then added a $196 nonitemizer charitable deduction, producing $283,854 of taxable income. The filer instead itemizes the larger $17,090.34 SALT deduction, producing $283,059.84."
-us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer implies that deductions or credits eliminated all tax, but the traced deductions leave $283,059.84 of taxable income and no nonrefundable credits to erase the resulting liability."
-us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $42,857 overtime premium even though it is already included in the stated $300,000 annual gross wages, omitted the alimony deduction, and used the standard deduction. It also submitted $68,513.50 despite its own arithmetic producing $85,241.65."
-us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,The model applied married-filing-jointly-style brackets and a child tax credit to a single filer with no child. It also invented a qualified business income deduction despite no listed qualified business income.
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the $16,100 standard deduction after counting only $13,834 of itemized deductions. The applicable itemized deduction is $17,090.34, so taxable income is $283,059.84 rather than $284,050."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $138,375 of unreimbursed employee business expenses from AGI and then produced tax arithmetic inconsistent with its own $146,775 taxable-income figure. Those expenses do not reduce this computation; AGI is $300,150.19 and taxable income is $283,059.84 after itemizing."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model added the $42,857 FLSA overtime premium to the stated $300,000 annual gross wages even though the prompt says gross wages already include overtime. This inflated gross income by $42,857, and its submitted $70,813 also contradicted its own calculated total of about $82,743."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model disallowed the trace’s $5,708.82 alimony deduction and invented $23,855 of mortgage interest from the listed mortgage balance despite the instruction that unlisted amounts are zero. It also imposed the wrong SALT limit, so neither its AGI nor its itemized deductions matched the required taxable-income computation."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model guessed that itemized deductions were between $30,000 and $40,000 instead of calculating the engine’s $17,090.34 amount. Its estimated taxable-income range and $72,700 result therefore do not follow from the household inputs or the applicable deduction comparison."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model rejected the $5,708.82 alimony adjustment and used an estimated $15,400 standard deduction. The required computation deducts alimony, then itemizes $17,090.34, yielding taxable income of $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 and chose an estimated $15,000 standard deduction instead of the $17,090.34 itemized deduction. It also incorrectly added the $900 Additional Medicare Tax to an income-tax output and ultimately submitted $76,500 despite deriving about $70,467."
+us,scenario_020,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted employer-sponsored insurance premiums from wages, deducted unreimbursed employee expenses above a 2% floor, and claimed a personal exemption under an inapplicable pre-TCJA reconstruction. The computation instead starts with the full $300,000 wages, deducts alimony above the line, and uses $17,090.34 of itemized deductions."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the listed insurance premiums and built an expired-law regular-tax calculation around unreimbursed employee expenses and a personal exemption. It then substituted an estimated AMT liability for the regular computation instead of taxing $283,059.84 under the applicable 2026 rules."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model omitted the $5,708.82 alimony deduction and described a $10,000 SALT limit and standard-deduction calculation. The correct path uses AGI of $300,150.19 and $17,090.34 of itemized deductions."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced wages by employer insurance premiums and restored the miscellaneous itemized deduction for unreimbursed employee expenses. Its resulting AMT pathway replaced the applicable regular-tax computation on $283,059.84 of taxable income."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model arrived at $291,761 of AGI by subtracting both employer insurance premiums and alimony from income, then claimed about $146,570 of itemized deductions using unreimbursed employee expenses. AGI is $300,150.19 and allowable itemized deductions are $17,090.34."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer insurance premiums from stated gross wages and deducted unreimbursed employee expenses above a 2% floor plus a personal exemption. Those reductions do not apply; taxable income is $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model reduced gross wages by $8,389 of employer insurance premiums and applied sunset-style miscellaneous itemized deductions and a personal exemption. The engine instead deducts $5,708.82 of alimony and $17,090.34 of itemized deductions."
+us,scenario_020,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model restored the deduction for unreimbursed employee business expenses, claimed a personal exemption, and used reverted pre-TCJA brackets. The applicable itemized deduction is only $17,090.34, leaving $283,059.84 taxable."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated unlisted mortgage interest and the $138,375 of unreimbursed employee business expenses as deductions large enough to erase taxable income. Mortgage interest is zero under the prompt and those employee expenses do not produce that deduction; taxable income remains $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted $31,472 is consistent with applying large expired-law itemized deductions rather than the trace’s $17,090.34 itemized amount. The required derivation leaves $283,059.84 taxable and produces $68,056.71."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model correctly reached approximately $300,150 of AGI but chose the $16,100 standard deduction. Itemized deductions are $17,090.34, reducing taxable income by an additional $990.16 to $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model states that it applied an employer-sponsored-insurance adjustment to the already stated annual gross wages. The $300,000 wage input is used without subtracting the $8,389 premium, after which alimony and $17,090.34 of itemized deductions produce $283,059.84 of taxable income."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model chose the standard deduction and added a $196 nonitemizer charitable deduction. The correct comparison selects $17,090.34 of itemized deductions, within which the applicable charitable treatment is already reflected, yielding $283,059.84 taxable income."
+us,scenario_020,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction and did not apply the trace’s deductible alimony adjustment. The computation requires AGI of $300,150.19 followed by $17,090.34 of itemized deductions."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an undifferentiated estimate based on roughly $306,000 of AGI and the standard deduction. It omitted the $5,708.82 alimony adjustment and the choice of $17,090.34 in itemized deductions."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted $132,372 of unreimbursed employee expenses, claimed a personal exemption, and applied reconstructed pre-TCJA brackets. The allowable itemized deductions total $17,090.34, not $146,402."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the alimony deduction and restored large miscellaneous employee-expense deductions before substituting an AMT calculation. The applicable computation uses $300,150.19 of AGI and $17,090.34 of itemized deductions without that expired-law detour."
+us,scenario_020,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted employer health premiums from gross wages and deducted unreimbursed employee expenses above a 2% floor under pre-TCJA rules. It also omitted the alimony adjustment; the correct taxable income is $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,inkling,llm_error,payroll_tax_base,False,"The model subtracted the $8,389 employer-sponsored insurance premium from the stated $300,000 gross-wage total and treated charity as an above-the-line adjustment. Gross wages are not reduced again, alimony supplies the above-the-line deduction, and the filer itemizes $17,090.34."
+us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_020,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model chose the standard deduction and then separately subtracted a $196 nonitemizer charitable amount, producing $283,854 of taxable income. The correct deduction is the single $17,090.34 itemized amount, producing $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer implies that deductions or credits erased the liability, but no listed deduction or nonrefundable credit does so. After the alimony adjustment and $17,090.34 of itemized deductions, $283,059.84 remains taxable."
+us,scenario_020,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model disallowed the $5,708.82 alimony deduction and chose the $16,100 standard deduction. The correct path deducts alimony and itemizes $17,090.34, lowering taxable income to $283,059.84."
+us,scenario_020,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $42,857 overtime premium by adding it to annual gross wages even though the prompt explicitly says gross wages include overtime. It also omitted the alimony deduction, and its submitted $68,513.50 contradicted its own calculated $85,241.65."
+us,scenario_020,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,The model applied married-filing-jointly-style brackets and a child tax credit to a single filer with no child. It also invented a qualified-business-income deduction despite no qualified business income being listed.
 us,scenario_020,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_020,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $16,689 total, then submitted $15,200, a number unsupported by either its unreduced-wage calculation or its alternative premium-reduction calculation. Its final numeric output discarded its own component arithmetic."
 us,scenario_020,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model repeatedly computed totals between $15,703.20 and $16,710.34 but submitted $23,119.50, which follows none of its stated calculations. It also double-counted the $42,857 overtime premium despite the prompt stating that gross wages already include overtime and used an obsolete $168,600 Social Security wage base."
@@ -1056,125 +1106,133 @@ us,scenario_021,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model
 us,scenario_021,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value for the requested output and therefore failed the required submission contract.
 us,scenario_021,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,508 taxable pension as Missouri taxable income, ignored the applicable deductions and 2026 taxable-income threshold, and applied an unsupported flat 1.9% rate directly to that pension. Missouri taxable income is reduced to zero before the rate calculation, yielding $0 of tax before refundable credits."
 us,scenario_021,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It included only the $8,829 real-estate tax in SALT and therefore understated itemized deductions as $24,222 instead of $29,654.81. Applying the senior and QBI deductions to that overstated taxable income produced $12,327 rather than taxing $74,633.30."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It improperly deducted $8,351 of unreimbursed employee expenses above the line, treated all Social Security as taxable, used head-of-household status, and selected a standard deduction instead of the optimal itemized deductions. Its stated bracket calculation also totals $10,564, not its submitted $11,316."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It treated the taxpayer as a qualifying surviving spouse and applied joint brackets and a joint standard deduction despite the absence of a qualifying child. The correct calculation uses single brackets and the larger $29,654.81 itemized deduction, followed by the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It invented $12,771 of mortgage interest from the outstanding mortgage balance and allowed the full $6,000 senior deduction instead of the $3,951.05 amount after the income phaseout. It also applied qualifying-surviving-spouse brackets without a qualifying child."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It correctly identified single treatment but its claimed $8,547 is incompatible with the single 2026 brackets it said it used; tax on its own approximately $78,017 figure is far higher. It also understated itemized deductions by counting only real-estate tax in SALT rather than the $14,807.14 total SALT deduction."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It incorrectly assumed TCJA sunset rules, restored unreimbursed-employee deductions and personal exemptions, invented mortgage interest, and used qualifying-surviving-spouse rates. The operative 2026 rules instead produce $74,633.30 of taxable income after the traced itemized, QBI, and senior deductions and apply single brackets."
-us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used a joint standard deduction and joint brackets solely from the surviving-spouse label, despite no qualifying child. It also failed to use the optimal $29,654.81 itemized deduction and omitted the phased senior and QBI deductions from its taxable-income derivation."
-us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied obsolete post-sunset rules by deducting unreimbursed employee expenses and a personal exemption. It also omitted the $3,951.05 senior deduction and used an incorrect itemized-deduction composition instead of the traced $29,654.81 amount."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It restored a personal exemption and the miscellaneous itemized deduction for unreimbursed employee expenses under an incorrect TCJA-sunset assumption. It omitted the $3,951.05 senior deduction and did not use the traced $29,654.81 itemized amount."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its answer does not identify a reproducible taxable-income derivation and refers to mortgage interest even though no mortgage-interest payment was supplied. The correct deductions yield taxable income of $74,633.30, whose single-bracket tax is $11,131.33."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It used pre-TCJA itemized deductions and personal exemptions that do not apply in this 2026 calculation. It also omitted the traced senior-deduction pathway and failed to derive taxable income of $74,633.30."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It incorrectly restored the unreimbursed-employee deduction and a personal exemption, while omitting the phased $3,951.05 senior deduction and $910 QBI deduction. Those errors replaced the traced $34,515.86 total deduction with an obsolete-law construction."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It understated itemized deductions as $25,393 and omitted the $3,951.05 senior deduction. The traced deductions are $29,654.81 itemized, $910 QBI, and $3,951.05 senior, reducing taxable income to $74,633.30."
-us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its unexplained $9,260 does not equal the 2026 single-bracket tax on the correctly derived $74,633.30 taxable income. The answer implies either a lower-rate filing schedule or excess deductions beyond the traced itemized, QBI, and senior deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It omitted the $3,951.05 senior deduction and understated SALT by including only $8,829 of real-estate tax, producing taxable income of $84,016.75 instead of $74,633.30. It also used 2025 rather than 2026 bracket thresholds."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It falsely treated the taxpayer as having no taxable wage income and asserted unspecified offsets that eliminate tax. Gross income is $109,149.16 and the allowed deductions leave $74,633.30 taxable, with no nonrefundable credits reducing the resulting $11,131.33 liability."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It subtracted unspecified nonrefundable credits despite no facts supporting such credits and supplied no deduction or bracket arithmetic. The traced taxable income of $74,633.30 generates $11,131.33 before refundable credits, with no such offset."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse joint brackets even though no qualifying child supports that filing status, and it introduced a personal exemption. The correct computation uses single brackets after the traced itemized, senior, and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Although it named the relevant deduction categories, its $10,247 does not result from the traced $29,654.81 itemized deduction, $3,951.05 senior deduction, $910 QBI deduction, and 2026 single brackets. Those inputs yield taxable income of $74,633.30 and tax of $11,131.33."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It explicitly used surviving-spouse joint-rate treatment without the required qualifying child. The applicable single schedule taxes the correctly deducted $74,633.30 of taxable income at $11,131.33."
-us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It explicitly applied a surviving-spouse tax schedule despite the absence of a qualifying child. It also failed to show the senior and QBI deductions needed to reach the traced $74,633.30 taxable income before applying single brackets."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It used a qualifying-surviving-spouse standard deduction and joint-rate brackets without a qualifying child. It also discarded the optimal itemized deduction, QBI deduction, and phased senior deduction that reduce taxable income to $74,633.30 under the single schedule."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied obsolete sunset-law treatment by deducting unreimbursed employee expenses and a personal exemption. It omitted the $3,951.05 senior deduction and used the wrong itemized-deduction components rather than the traced $29,654.81 total."
-us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It taxed 100% of Social Security instead of the $17,624.75 taxable portion, overstating AGI by $3,110.25. It also restored unreimbursed-employee deductions under an incorrect sunset assumption and omitted the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It invented $901 of rental depreciation and a 0.5% floor for charitable contributions, neither of which follows from the supplied inputs or traced computation. It also understated SALT and consequently failed to reach the $29,654.81 itemized deduction and $74,633.30 taxable income."
-us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It returned no numeric value or explanation for the requested output, so there is no substantive tax computation to evaluate."
-us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It estimated rather than derived California income tax for SALT, understated total SALT relative to the traced $14,807.14, and omitted the $910 QBI deduction. These errors left taxable income at $78,275.67 instead of $74,633.30."
-us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction and unidentified nonrefundable credits eliminate liability, despite $109,149.16 of gross income and no supported credits. The allowed deductions leave $74,633.30 taxable and therefore a positive $11,131.33 tax."
-us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It arithmetically overstated AGI as $126,773.50 by effectively adding taxable Social Security twice; the listed components total $109,148.75. It then invented mortgage interest from the loan balance, used qualifying-surviving-spouse rates without a qualifying child, and omitted the senior and QBI deductions."
-us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It treated deductions such as charity, medical costs, taxes, employee expenses, and estimated mortgage interest as both income-reducing items and nonrefundable credits, producing negative taxable income without support. The traced allowed deductions total $34,515.86 and leave $74,633.30 taxable, with no credits eliminating the tax."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It included only $8,829 of property tax in SALT and used the stated $15,393 contribution rather than the computed itemized amounts, understating itemized deductions as $24,222 instead of $29,654.81. With the $910 QBI and $3,951.05 senior deductions, taxable income is $74,633.30, not $80,066."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It incorrectly deducted $8,351 of unreimbursed employee expenses above the line, treated the taxpayer as head of household, included all Social Security rather than only $17,624.75, and used the standard deduction. The applicable computation uses single rates and $34,515.86 of itemized, QBI, and senior deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"It treated the taxpayer as a qualifying surviving spouse and applied the joint standard deduction and joint brackets despite there being no qualifying child. The computation instead uses single brackets on $74,633.30 after the larger itemized-deduction pathway, QBI deduction, and phased senior deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It invented $12,771 of mortgage interest from the outstanding mortgage balance, used qualifying-surviving-spouse brackets, and allowed the full $6,000 senior deduction without its AGI phaseout. Mortgage interest was not supplied, and the senior deduction is $3,951.05."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its stated deductions lead to roughly $78,017 of taxable income under single rates, but it then reported $8,547, a result inconsistent with those rates and its own calculation. It also understated itemized deductions by omitting the full computed SALT component; taxable income is $74,633.30 and tax is $11,131.33."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It applied a nonexistent 2026 TCJA sunset regime, restored miscellaneous employee deductions and personal exemptions, invented mortgage interest, and used qualifying-surviving-spouse rates. The operative 2026 rules retain the current bracket structure, disallow those employee expenses, and use the traced $34,515.86 deduction total under single filing status."
+us,scenario_022,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"It used the joint standard deduction and joint brackets solely from the surviving-spouse fact, even though no qualifying child supports qualifying-surviving-spouse status. It also omitted the phased senior and QBI deductions from the operative itemized pathway."
+us,scenario_022,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applied expired-law unreimbursed-employee deductions and a personal exemption while omitting the $3,951.05 senior deduction and $910 QBI deduction. The correct deduction stack is $29,654.81 of itemized deductions plus those two deductions, producing $74,633.30 of taxable income under the 2026 single brackets."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It restored a 2%-floor employee-business-expense deduction and a personal exemption under an obsolete post-sunset assumption. Those deductions do not apply; the relevant total is $29,654.81 itemized plus $910 QBI and $3,951.05 senior deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its generic explanation does not identify the deduction stack supporting its number. The traced computation itemizes $29,654.81, adds the $910 QBI and $3,951.05 senior deductions, and taxes $74,633.30 under the single schedule; its result does not follow that calculation."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly used pre-TCJA itemized deductions and personal exemptions, which are not part of the operative 2026 calculation. It also failed to apply the traced phased senior and QBI deductions within the $34,515.86 total deduction stack."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It deducted unreimbursed employee expenses after a 2% floor and a personal exemption under an obsolete sunset regime. The valid deductions are $29,654.81 itemized, $910 QBI, and $3,951.05 senior, leaving $74,633.30 taxable."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It omitted the $3,951.05 senior deduction and understated itemized deductions by using only $25,393 rather than $29,654.81. Those omissions inflated taxable income to $82,846 instead of $74,633.30."
+us,scenario_022,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its unexplained $9,260 does not result from applying the 2026 single brackets to the traced taxable income. After $34,515.86 of total deductions, taxable income is $74,633.30 and the bracket calculation yields $11,131.33."
+us,scenario_022,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It limited itemized deductions to property tax and stated charitable gifts, omitting the additional SALT amount included in the $29,654.81 traced itemized deduction and the $3,951.05 senior deduction. It also applied 2025 brackets instead of the 2026 brackets."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It incorrectly asserted that there was no taxable wage income and that deductions or credits eliminated the liability. Wages are $52,635, AGI is $109,149.16, and no nonrefundable credit offsets the $11,131.33 tax produced from $74,633.30 of taxable income."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"Its $1,237 answer implies nearly all regular tax was removed by nonrefundable credits, but the facts provide no such usable credits. The full bracket tax on $74,633.30 remains $11,131.33 before refundable credits."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"It used surviving-spouse joint brackets and a personal exemption rather than single filing status and the operative deduction rules. The correct taxable income is $74,633.30 after itemized, QBI, and phased senior deductions, taxed on the single schedule."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Although it named the senior and QBI deductions, its reported tax does not follow from the complete traced deduction stack and 2026 single brackets. Those steps yield $74,633.30 of taxable income and $11,131.33 of tax."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"It applied surviving-spouse joint-rate treatment and an age-based standard deduction. With no qualifying child, the taxpayer uses single status, itemizes, and separately receives the phased $3,951.05 senior deduction and $910 QBI deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"It used the surviving-spouse tax schedule rather than the single schedule applicable without a qualifying child. The charitable and SALT deductions form part of $29,654.81 of itemized deductions, followed by the QBI and phased senior deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"It used a qualifying-surviving-spouse standard deduction and joint brackets. The taxpayer instead uses single rates and the superior itemized pathway totaling $29,654.81, plus $910 QBI and $3,951.05 senior deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It applied a post-TCJA-sunset personal exemption and deductible unreimbursed employee expenses, neither of which belongs in the 2026 computation. It omitted the applicable phased senior deduction and used an incorrect itemized total."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"It correctly selected single status and included the senior and QBI deductions, but understated itemized deductions as $26,196 instead of $29,654.81. That left taxable income $3,458.70 too high and overstated the tax."
+us,scenario_022,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It treated all $20,735 of Social Security as taxable instead of $17,624.75 and deducted unreimbursed employee expenses under an obsolete 2%-floor rule. It also omitted the $3,951.05 senior and $910 QBI deductions."
+us,scenario_022,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It invented $901 of rental depreciation and a 0.5%-of-AGI reduction to charitable deductions, neither of which follows from the inputs. The traced AGI is $109,149.16 and the complete deduction total is $34,515.86, producing $74,633.30 taxable income."
+us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the required structured answer could not be parsed."
+us,scenario_022,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It omitted the $910 QBI deduction and understated itemized deductions as $26,922 rather than $29,654.81. Those omissions raised taxable income from $74,633.30 to $78,275.67."
+us,scenario_022,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It asserted that the standard deduction and unspecified credits reduced taxable income or tax to zero. AGI of $109,149.16 remains $74,633.30 after all applicable deductions, and no nonrefundable credit eliminates the resulting tax."
+us,scenario_022,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"It used qualifying-surviving-spouse joint brackets and the joint standard deduction, allowed the full $6,000 senior deduction without its phaseout, and omitted the $910 QBI deduction. The taxpayer instead uses single status, itemizes, and receives a $3,951.05 senior deduction."
+us,scenario_022,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It double-counted taxable Social Security when summing AGI, producing $126,773.50 instead of $109,149.16, then invented mortgage interest from the loan balance. It also used qualifying-surviving-spouse rates rather than single rates."
+us,scenario_022,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It converted deductions such as charity, taxes, medical costs, employee expenses, and estimated mortgage interest into nonrefundable credits and asserted they erased the tax. These items are not credits, several are not deductible as claimed, and the valid deductions leave $74,633.30 taxable with $11,131.33 due."
 us,scenario_022,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_022,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare enrollment plus age 77 as sufficient for Medicaid eligibility. Medicare status does not establish Medicaid eligibility, and this person receives no SSI and has medicaid_category NONE, so no California Medicaid pathway applies."
 us,scenario_022,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate instead of the 2026 rate of 1.3%, understating SDI by $52.64. It then submitted $4,557.61 even though its own stated components totaled $4,658.20."
-us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model excluded California SDI by asserting that no mandatory California employee payroll tax applies. Payroll_tax includes the mandatory 1.3% SDI contribution of $684.26, and its submitted value also fails to match its stated federal-tax total."
-us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. This calculated $631.62 rather than $684.26 and understated payroll tax by $52.63.
-us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model explicitly excluded California SDI even though the requested payroll_tax includes mandatory employee state payroll taxes. The 1.3% SDI contribution adds $684.26 to federal FICA.
-us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model omitted California’s mandatory employee SDI contribution, incorrectly limiting payroll tax to Social Security and Medicare. It also rounded its stated $4,026.58 federal total down to $4,026.00 without justification."
-us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,The model used the obsolete 1.1% California SDI rate instead of the 2026 rate of 1.3%. The correct state contribution is $684.26 rather than $578.99.
-us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly treated California SDI as nonmandatory and omitted the $684.26 employee contribution. Its submitted $3,264.37 also contradicts its own correctly stated $4,026.58 combined federal FICA calculation."
-us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model’s $579 California SDI component applies a 1.1% rate. The 2026 rate is 1.3%, producing $684.26 and total payroll tax of $4,710.83."
-us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model applied a 1.1% California SDI rate instead of 1.3%. This understated the mandatory state payroll contribution by $105.27.
-us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model calculated only Social Security and Medicare and omitted California SDI. The mandatory 1.3% employee contribution adds $684.26 to the federal payroll taxes.
-us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model used a 1.1% California SDI rate instead of the 2026 rate of 1.3%. Its rounded $579 state component must be $684.26.
-us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,The model applied 1.1% to California SDI rather than the 2026 rate of 1.3%. That reduced the state component from $684.26 to about $579.
-us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a 0.9% California SDI rate instead of 1.3%. The state contribution is $684.26, not $473.72."
-us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted total implies California SDI of about $473.71, equal to 0.9% of wages. The 2026 SDI rate is 1.3%, yielding $684.26."
-us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate rather than 1.3%. The resulting $631.62 state tax is $52.64 below the required $684.26.
-us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,The model incorrectly asserted that California has no employee state payroll tax and omitted SDI. California’s mandatory 1.3% employee SDI contribution is $684.26.
-us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model omitted the $684.26 California SDI contribution and also miscomputed federal FICA: 7.65% of $52,635 is $4,026.58, not $3,308.00. The required three-component total is $4,710.83."
-us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,The model estimated California SDI at 1.2% instead of applying the 2026 rate of 1.3%. This understated the state contribution by $52.64.
-us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model stopped after calculating federal Social Security and Medicare at 7.65% and omitted California SDI. The mandatory state contribution adds $684.26.
-us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model calculated only Social Security and Medicare and omitted mandatory California SDI. Applying 1.3% to $52,635 adds $684.26."
-us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,The model used a 1.2% California SDI rate instead of the 2026 rate of 1.3%. Its rounded $632 state component must be $684.26.
-us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,The model used a projected 1.2% California SDI rate rather than the applicable 2026 rate of 1.3%. This produced about $632 instead of $684.26.
-us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,"The model approximated California SDI using 1.2% instead of the 2026 rate of 1.3%. The correct state component is $684.26, not about $632."
-us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required output contract."
-us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate rather than the 2026 rate of 1.3%. The state contribution is $684.26 instead of $631.62.
-us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model included only Social Security and Medicare and omitted California’s mandatory employee SDI tax. The missing 1.3% contribution is $684.26.
-us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,The model included California SDI but applied an approximate 1.1% rate instead of the 2026 rate of 1.3%. This calculated $578.99 rather than $684.26.
-us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,The model incorrectly stated that no mandatory California employee state payroll tax applies and omitted SDI. The required 1.3% SDI contribution is $684.26.
+us,scenario_022,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate instead of the 2026 rate of 1.3%, and then submitted $4,557.61 despite its own components summing to $4,658.20. California SDI is $684.26, bringing total payroll tax to $4,710.83."
+us,scenario_022,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly excluded mandatory California SDI, which contributes $684.26 at 1.3% of wages. It also misstated Medicare tax and submitted a number inconsistent with its own $4,026.08 total."
+us,scenario_022,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. SDI is $684.26 rather than $631.62, so total payroll tax is $4,710.83."
+us,scenario_022,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model excluded California SDI even though the requested payroll-tax definition expressly includes mandatory employee state payroll taxes. The required 1.3% SDI contribution is $684.26.
+us,scenario_022,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model omitted the mandatory California SDI contribution of $684.26. It also rounded its own federal FICA result of $4,026.58 down to $4,026.00 without justification."
+us,scenario_022,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an outdated 1.1% California SDI rate instead of the 2026 rate of 1.3%. Applying 1.3% produces $684.26 of SDI and $4,710.83 of total payroll tax."
+us,scenario_022,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model wrongly characterized California SDI as nonmandatory and excluded its $684.26 employee contribution. Its submitted $3,264.37 also contradicts its own combined federal FICA calculation of $4,026.58 by effectively dropping Medicare from the final answer."
+us,scenario_022,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI amount of about $579 instead of the 2026 1.3% amount of $684.26. Its rounded federal components further obscured the required total of $4,710.83."
+us,scenario_022,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate instead of the 2026 rate of 1.3%. The state contribution is $684.26, not $578.99."
+us,scenario_022,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model calculated only Social Security and Medicare and omitted mandatory California SDI. The missing state contribution is $684.26 at 1.3% of $52,635."
+us,scenario_022,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model used a 1.1% California SDI rate instead of the 2026 rate of 1.3%. This understated SDI by about $105.27 before whole-dollar rounding.
+us,scenario_022,payroll_tax,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate instead of the applicable 1.3% rate. California SDI is $684.26, making total payroll tax $4,710.83."
+us,scenario_022,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,The model applied a 0.9% California SDI rate instead of the 2026 rate of 1.3%. The correct state contribution is $684.26 rather than $473.72.
+us,scenario_022,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted total implies a 0.9% California SDI rate on top of federal FICA. The 2026 SDI rate is 1.3%, producing $684.26 and a $4,710.83 total."
+us,scenario_022,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. This understated the state contribution by $52.64.
+us,scenario_022,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,The model incorrectly asserted that California has no employee state payroll tax and omitted the mandatory SDI contribution. California SDI adds $684.26 to federal FICA.
+us,scenario_022,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model said it added 6.2% Social Security and 1.45% Medicare, but those taxes alone equal $4,026.58 rather than $3,308.00. It also omitted the $684.26 mandatory California SDI contribution."
+us,scenario_022,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated California SDI at 1.2% instead of applying the 2026 rate of 1.3%. The correct SDI amount is $684.26, not $631.62."
+us,scenario_022,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model included only the 7.65% federal FICA taxes and omitted mandatory California SDI. The missing 1.3% state contribution is $684.26.
+us,scenario_022,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model calculated only Social Security and Medicare and omitted the mandatory California SDI employee tax. SDI contributes $684.26 at the 2026 rate of 1.3%.
+us,scenario_022,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate instead of the applicable 1.3% rate. The correct SDI contribution is $684.26 and the exact total is $4,710.83."
+us,scenario_022,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. It therefore used $631.62 rather than $684.26 for the state component.
+us,scenario_022,payroll_tax,grok-build-0.1,llm_error,thresholds_rates,False,The model used a projected 1.2% California SDI rate instead of the 2026 rate of 1.3%. It also rounded each component to whole dollars instead of preserving the engine's cent-level amounts.
+us,scenario_022,payroll_tax,inkling,llm_error,thresholds_rates,False,The model estimated California SDI at about 1.2% rather than applying the 2026 rate of 1.3%. The state tax is $684.26 rather than about $632.
+us,scenario_022,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required structured-output contract."
+us,scenario_022,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,The model applied a 1.2% California SDI rate instead of the 2026 rate of 1.3%. This understated the state component from $684.26 to $631.62.
+us,scenario_022,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model stopped after Social Security and Medicare and omitted mandatory California SDI. The missing state payroll tax is $684.26.
+us,scenario_022,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate rather than the 2026 rate of 1.3%. Whole-dollar component rounding compounded the understatement, while the exact total is $4,710.83."
+us,scenario_022,payroll_tax,qwen-3.7-max,llm_error,thresholds_rates,False,The model included California SDI but applied a 1.1% rate instead of the 2026 rate of 1.3%. The correct SDI contribution is $684.26 rather than $578.99.
+us,scenario_022,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model incorrectly excluded mandatory California employee payroll taxes. California SDI contributes $684.26 at 1.3% of the same $52,635 wage base."
 us,scenario_022,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model treated the surviving-spouse flag as qualifying-surviving-spouse status and applied joint-equivalent brackets and deductions despite the absence of a qualifying child. It also deducted unreimbursed employee expenses, whereas the traced California itemized deduction is $23,676.67, producing $67,847.75 of taxable income."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a $5,320 income-tax credit from California's Senior Property Tax Postponement program. That program postpones secured property taxes through a state lien; it does not offset California individual income tax, whose traced nonrefundable credits total $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model applied married-joint/qualifying-surviving-spouse brackets and exemption credits solely from the surviving-spouse fact. With no qualifying child, the calculation uses the applicable single schedule on $67,847.75 and subtracts $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned qualifying-surviving-spouse filing status and joint brackets. It also estimated unlisted mortgage interest despite the instruction that unlisted amounts are zero; the engine instead allows $23,676.67 of itemized deductions and applies the single schedule."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the correct general single-filer pathway but overstated deductible contributions by treating all $15,393 of listed donations as deductible. The traced charitable deduction is $14,847.67, total itemized deductions are $23,676.67, and the resulting taxable income is $67,847.75 rather than $67,302."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated a surviving spouse without a qualifying child as a qualifying surviving spouse and used joint brackets plus a joint personal exemption credit. The applicable single schedule produces $2,752.58 before the traced $312.93 of aged/personal credits."
-us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model automatically converted surviving-spouse status into joint-equivalent filing status and then estimated tax from an effective rate. The absence of a qualifying child requires the applicable single schedule, and the senior exemption is a separate nonrefundable credit rather than embedded in the brackets."
-us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $6,168 of unreimbursed employee expenses and thereby reduced taxable income to $61,134. The traced allowed itemized deductions are $23,676.67, yielding taxable income of $67,847.75 before applying the California schedule."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included unreimbursed employee expenses in $30,742.52 of itemized deductions. The allowed itemized amount is $23,676.67, so taxable income is $67,847.75 and gross tax is $2,752.58 before $312.93 of credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $1,175 does not follow the California single-filer schedule on the traced $67,847.75 of taxable income. That schedule yields $2,752.58 before $312.93 of nonrefundable credits, not the model's unexplained low estimate."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model overstated itemized deductions at $30,743 by including employee business expenses. California's traced itemized deduction is $23,676.67, which leaves $67,847.75 taxable rather than $60,781."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $60,781 using $30,743 of itemized deductions, including an inadmissible employee-expense amount. The traced deduction is $23,676.67 and taxable income is $67,847.75."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model expressly included employee business expenses in $30,743 of California itemized deductions. The calculation allows $23,676.67, producing $67,847.75 of taxable income and $2,752.58 of gross tax."
-us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer failed to carry the traced calculation through both stages: the California schedule produces $2,752.58 on $67,847.75, then $312.93 of aged/personal nonrefundable credits must be subtracted. Its $2,652 figure reflects neither the gross tax nor the post-credit amount."
-us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented $24,556 of rental income from a stated $4,550 input and fabricated mortgage-interest and Social Security amounts. Using the listed income gives federal AGI of $109,149.16, California AGI of $91,524.41, and taxable income of $67,847.75."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly concluded that age and deductions eliminate the liability. The deductions leave $67,847.75 taxable, generating $2,752.58 of gross California tax, and the $312.93 nonrefundable credits reduce it only to $2,439.65."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The unexplained $1,036 substantially understates the tax produced by the California schedule. On $67,847.75 of taxable income, gross tax is $2,752.58 and the only traced reduction is $312.93 of nonrefundable credits."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used joint/qualifying-surviving-spouse rates even though no qualifying child is present, and it overstated both itemized deductions and exemption credits. The applicable single schedule and traced $312.93 credit produce $2,439.65."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model did not correctly subtract the traced nonrefundable aged/personal credits from scheduled tax. Gross tax is $2,752.58 and credits are $312.93, so the post-credit amount is $2,439.65 rather than $2,893."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse rates without the required qualifying child and also included employee expenses among itemized deductions. The engine uses the applicable single schedule and $23,676.67 of itemized deductions."
-us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's $1,011 result applies the favorable surviving-spouse tax treatment solely from marital history. Without a qualifying child, the applicable single schedule yields $2,752.58 before $312.93 of exemption credits."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model replaced the required computation with a zero estimate. The listed facts determine $67,847.75 of California taxable income, $2,752.58 of scheduled tax, and $2,439.65 after nonrefundable credits."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included employee business expenses in $30,743 of itemized deductions and claimed only a personal exemption credit. Allowed itemized deductions are $23,676.67, and both applicable aged/personal credits total $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $6,521 of unreimbursed employee expenses, reducing taxable income to $60,781. It also failed to subtract the traced $312.93 of nonrefundable aged/personal exemption credits; correct taxable income is $67,847.75."
-us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated California AGI and deducted unreimbursed employee expenses to reach only $59,862 of taxable income. The traced figures are $91,524.41 of California AGI, $23,676.67 of itemized deductions, and $67,847.75 of taxable income."
-us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the response could not be parsed or evaluated as a substantive calculation."
-us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model treated the full $15,393 of donations as deductible and used $24,222 of itemized deductions. The traced charitable deduction is $14,847.67 and total itemized deductions are $23,676.67; it also omitted part of the $312.93 aged/personal credit total."
-us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly treated pensions, deductions, and the senior exemption as eliminating California taxable income. The calculation leaves $67,847.75 taxable, and the senior and personal credits reduce $2,752.58 of tax by only $312.93."
-us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an inverted and incorrect California rate schedule, beginning with 4% and progressing to 9.3%, rather than applying the actual lower graduated brackets from the bottom. It also used the standard deduction even though $23,676.67 of allowed itemized deductions is larger."
-us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly claimed deductions and senior credits fully offset income and tax. Deductions leave $67,847.75 taxable, while the $312.93 aged/personal credits reduce $2,752.58 of gross tax only to $2,439.65."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model treated the surviving-spouse fact as automatic entitlement to qualifying-surviving-spouse joint brackets and a joint standard deduction, even though the household facts do not establish that filing status. It also deducted $6,521 of unreimbursed employee expenses, which are absent from the allowable $23,676.67 California deductions, and its final $3,162 contradicts its own approximately $980 computation."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $134,529 of federal taxable income and carried $106,979 into California taxable income instead of deriving $67,847.75. It then fabricated a roughly $5,320 income-tax credit from California's Senior Property Tax Postponement program, which postpones property-tax payment and is not a nonrefundable California income-tax credit."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse joint brackets and joint exemption credits solely from the surviving-spouse label. The applicable single schedule taxes $67,847.75 at $2,752.58 before $312.93 of exemption credits, rather than producing approximately $1,400 under joint brackets."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model automatically assigned MFJ-equivalent surviving-spouse brackets and also invented mortgage interest from the mortgage balance despite the instruction that unlisted expenses are zero. Its stated bracket arithmetic gives $1,228 before credits and therefore does not support its submitted $1,856."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the full $15,393 of stated donations rather than the $14,847.67 deductible charitable amount, producing $67,302 instead of $67,847.75 of taxable income. It also misstated the scheduled tax and credits: the correct steps are $2,752.58 of tax followed by $312.93 of nonrefundable credits."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model used MFJ-equivalent brackets and a joint personal exemption credit merely because the person is a surviving spouse. Applying the single schedule to $67,847.75 yields $2,752.58, and only $312.93—not the model's $588—reduces that tax."
+us,scenario_022,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-widow joint brackets without the facts required for that filing status and estimated an effective rate instead of calculating the schedule. It also treated the senior exemption credit as embedded in the brackets, whereas the $312.93 aged/blind and personal credits are separately subtracted from $2,752.58."
+us,scenario_022,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $6,168 of unreimbursed employee expenses after a 2% floor, but those expenses are not part of the allowable $23,676.67 California deductions. That error reduced taxable income to $61,134 instead of $67,847.75 and caused it to understate scheduled tax."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used $30,742.52 of itemized deductions by including employee business expenses, rather than the allowable $23,676.67. This lowered taxable income from $67,847.75 to $60,781.48, so its otherwise single-schedule approach began from the wrong tax base."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $1,175 answer does not implement the traced tax base and schedule. California taxable income is $67,847.75, producing $2,752.58 before $312.93 of nonrefundable credits, not a low-income estimate based only on a generic standard deduction."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model deducted $30,743 by including unreimbursed employee expenses, reducing taxable income to $60,781 instead of $67,847.75. The correct allowable deductions are $23,676.67, after which the single schedule and $312.93 credits yield $2,439.65."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly included unreimbursed employee expenses in $30,743 of itemized deductions. This understated taxable income by about $7,067 and also used $288 rather than $312.93 of personal and aged/blind exemption credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model expressly included employee business expenses in $30,743 of California itemized deductions. The allowable deductions are $23,676.67, leaving $67,847.75 taxable and $2,752.58 of scheduled tax before $312.93 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The terse calculation failed to subtract the full $312.93 of personal and aged/blind nonrefundable exemption credits from the $2,752.58 scheduled tax. The resulting liability is $2,439.65, not $2,652."
+us,scenario_022,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model transformed the listed $4,550 of rental income into $24,556 and invented mortgage interest and a $1,000 taxable Social Security amount, violating the supplied annual inputs. Those inventions produced a fictitious AGI and tax base instead of California AGI of $91,524.41 and taxable income of $67,847.75."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age and deductions as fully offsetting California tax. The deductions leave $67,847.75 taxable, and the $312.93 aged/blind and personal credits offset only part of the $2,752.58 scheduled tax."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The answer does not show a calculation supporting $1,036. The required sequence is $91,524.41 of California AGI, $23,676.67 of deductions, $2,752.58 of scheduled tax, and $312.93 of nonrefundable credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used joint or qualifying-surviving-spouse brackets and $486 of credits, neither of which follows from this one-person tax unit. It also deducted employee expenses to reach $60,781 rather than using $67,847.75 of taxable income and $312.93 of credits."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model's $2,893 exceeds the $2,752.58 tax calculated by the California schedule before any credits, so it applied the rate schedule or taxable base incorrectly. The applicable $312.93 personal and aged/blind credits then reduce liability to $2,439.65."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse rates and included employee expenses among allowable deductions. The facts support the single schedule, and employee expenses are absent from the $23,676.67 deduction total, leaving $67,847.75 taxable."
+us,scenario_022,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's surviving-spouse tax of $1,011 reflects joint-equivalent treatment unsupported by the household facts. The applicable single schedule produces $2,752.58 before both the personal and aged/blind credits totaling $312.93."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model defaulted to zero instead of performing the calculation. The supplied facts determine $67,847.75 of California taxable income and a positive $2,439.65 liability after nonrefundable credits."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included unreimbursed employee expenses in $30,743 of itemized deductions, lowering taxable income to $60,781 instead of $67,847.75. It also subtracted only a personal exemption credit, omitting the full $312.93 combined personal and aged/blind credits."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model deducted $6,168 of unreimbursed employee expenses after a 2% floor, producing $61,134 of taxable income. Those expenses are excluded from the traced $23,676.67 deduction total, and the correct taxable income is $67,847.75."
+us,scenario_022,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $6,521 of employee business expenses and therefore used $60,781 rather than $67,847.75 of taxable income. It then failed to subtract the applicable $312.93 personal and aged/blind nonrefundable credits."
+us,scenario_022,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $90,623 California AGI and deducted employee expenses to reach only $59,862 of taxable income. California AGI is $91,524.41 and allowable deductions are $23,676.67, yielding $67,847.75 taxable; it also omitted the aged/blind portion of the $312.93 credits."
+us,scenario_022,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric value or explanation for the requested output, so the required structured answer could not be parsed."
+us,scenario_022,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the full $15,393 stated donations rather than the $14,847.67 deductible charitable amount, yielding $67,302 instead of $67,847.75 taxable income. It also subtracted only about $144 of exemption credit rather than the full $312.93 personal and aged/blind credits."
+us,scenario_022,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that pensions and deductions eliminate California taxable income. California AGI remains $91,524.41 after the Social Security subtraction, and $23,676.67 of deductions leaves $67,847.75 taxable."
+us,scenario_022,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that California taxes federally taxable Social Security, retaining $17,624.75 that California subtracts from federal AGI. It also used MFJ brackets, while the traced calculation uses $91,524.41 of California AGI and the single schedule."
+us,scenario_022,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model discarded the larger $23,676.67 itemized deduction in favor of an estimated standard deduction and then applied invented rate bands beginning at 4%, 6%, 8%, and 9.3% rather than California's actual progressive schedule. It also fabricated approximately $639.40 of credits instead of the applicable $312.93."
+us,scenario_022,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions and the senior exemption credit as eliminating all taxable income and tax. The deductions leave $67,847.75 taxable, and the $312.93 credits reduce—but do not eliminate—the $2,752.58 scheduled tax."
 us,scenario_022,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model invented an AOTC from enrollment facts despite the required qualified education expenses being zero, and it allowed a Saver's Credit without reducing eligible contributions by the $8,000 retirement distribution. Neither credit offsets the $643.43 tentative tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from gross income and used an incorrect $14,600 standard deduction instead of $16,100. Including the distribution yields $6,434.34 of taxable income and $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model applied the nonrefundable AOTC solely from student eligibility facts. Qualified education expenses are zero under the prompt, so the AOTC is zero and the $643.43 tentative tax remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated enrollment, credential, and 1098-T facts as establishing a maximum AOTC despite zero listed qualified education expenses. The AOTC is zero, leaving $643.43 after the ordinary tax calculation."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model subtracted an AOTC nonrefundable portion without any qualified education expenses. Student status alone generates no AOTC, so the $643.43 tentative liability is not offset."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the 2026 single standard deduction as $15,350 rather than applying the $16,100 amount. The correct deduction leaves $6,434.34 taxable and produces $643.43 at 10%."
-us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model relied on a nonrefundable AOTC to eliminate the residual tax even though no qualified education expenses were listed. It also misstated $17,443 plus $8,000 as $26,443; the proper income and deduction calculation yields $643.43 with no AOTC offset."
-us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model separately deducted $8,089 of employer-sponsored insurance premiums from wages even though the employment-income computation only excludes the $2,778.48 traditional 401(k) contribution. It then used an incorrect $8,350 standard deduction; AGI is $22,534.34 and the applicable standard deduction is $16,100."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model allowed a $1,000 Saver's Credit without applying the rule that recent retirement distributions reduce eligible retirement contributions. The $8,000 taxable 403(b) distribution exceeds the relevant contributions, leaving no Saver's Credit and $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its zero-taxable-income conclusion omits or neutralizes taxable income that remains after deductions. The $8,000 taxable distribution helps produce $22,534.34 of AGI, and subtracting the $16,100 standard deduction leaves $6,434.34 taxable."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly used the Saver's Credit to eliminate the liability. The $8,000 retirement distribution reduces eligible retirement contributions to zero for the credit, so $643.43 remains after nonrefundable credits."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The stated $14,446 AGI improperly subtracts the $8,089 employer-sponsored insurance premiums separately from wages. It also applies a Saver's Credit despite the offset from the $8,000 retirement distribution; the proper calculation leaves $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model subtracted a $200 Saver's Credit without applying the retirement-distribution offset. The $8,000 taxable 403(b) distribution exceeds eligible contributions, so the Saver's Credit is zero and the tax remains $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model concluded that deductions eliminate all taxable income, thereby omitting the effect of the $8,000 taxable 403(b) distribution. AGI of $22,534.34 less the $16,100 standard deduction leaves $6,434.34 taxable."
-us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly denied both credits but used an estimated $15,400 standard deduction instead of the 2026 single-filer amount of $16,100. That $700 difference reduces taxable income to $6,434.34 and tax to $643.43."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model used an AOTC to eliminate the remaining tax despite zero listed qualified education expenses. The student-status facts do not create creditable expenses, so no AOTC reduces the $643.43 liability."
-us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model partially offset the tentative tax with a nonrefundable AOTC despite zero qualified education expenses. No AOTC applies, so the full $643.43 ordinary tax remains."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated wages below the standard deduction as dispositive and failed to include the $8,000 taxable 403(b) distribution in the taxable-income calculation. Including it leaves $6,434.34 taxable and $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed a post-TCJA reversion to an approximately $8,500 standard deduction plus a personal exemption. The applicable 2026 schedule instead provides a $16,100 single standard deduction, yielding $6,434.34 of taxable income and $643.43 of tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied an obsolete-law reversion to an $8,500 standard deduction and a 15% second bracket. The 2026 calculation uses the $16,100 standard deduction, leaving all $6,434.34 of taxable income in the 10% bracket."
-us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $8,089 of employer-sponsored insurance premiums again when computing AGI, reducing its AGI to $14,446. The correct AGI is $22,534.34, so the $16,100 standard deduction does not eliminate taxable income."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model invented an AOTC from enrollment facts despite $0 of listed qualified education expenses and allowed a Saver's Credit without reducing eligible contributions by the $8,000 retirement distribution. Neither credit offsets the $643.43 tentative tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from gross income and used an incorrect $14,600 standard deduction instead of $16,100. Including the distribution yields $22,534.34 of AGI and $6,434.34 of taxable income."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model awarded the nonrefundable AOTC solely from student eligibility facts. With no qualified education expenses listed, the AOTC is $0 and the $643.43 tentative tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated possession of a 1098-T and qualifying enrollment as sufficient for the maximum AOTC. The credit calculation requires qualified education expenses, which are $0 under the prompt's unlisted-input rule, so no AOTC offsets the tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model applied up to $1,500 of nonrefundable AOTC without any listed qualified education expenses. The AOTC is $0, leaving the full $643.43 tax on taxable income."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly denied the AOTC but estimated the 2026 single standard deduction as $15,350. The applicable deduction is $16,100, leaving $6,434.34 taxable and $643.43 of tax rather than $719."
+us,scenario_023,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model misstated total income and then invoked an AOTC to eliminate the residual tax despite no listed qualified education expenses. After the valid deductions and $16,100 standard deduction, $6,434.34 remains taxable and no education credit applies."
+us,scenario_023,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,089 employer-sponsored insurance premium from the employee's income and then used an erroneous $8,350 standard deduction. The trace includes $14,664.17 of employment income after the 401(k) deferral, not after another subtraction for employer premiums, and applies the $16,100 standard deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model allowed a $1,000 Saver's Credit without applying the statutory reduction for retirement distributions during the testing period. The $8,000 taxable 403(b) distribution exceeds the relevant contributions, reducing eligible Saver's Credit contributions to zero."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's zero-taxable-income shortcut omits part of the taxable income or overstates deductions. The correct sequence produces $22,534.34 of AGI, then $6,434.34 of taxable income after the $16,100 standard deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly used the Saver's Credit to eliminate the tax. The $8,000 retirement distribution reduces eligible retirement contributions to zero, while the standard deduction alone still leaves $6,434.34 taxable."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model derived $14,446 of AGI by subtracting the $8,089 employer-sponsored premium from income and then incorrectly applied a Saver's Credit. AGI is $22,534.34, and the $8,000 retirement distribution eliminates contributions eligible for that credit."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model subtracted a $200 Saver's Credit from tentative tax. The $8,000 retirement distribution exceeds the retirement contributions counted for the credit, so eligible contributions are zero and the entire $643.43 tax remains."
+us,scenario_023,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions reduce taxable income to zero. AGI is $22,534.34 and the $16,100 standard deduction leaves $6,434.34 taxable."
+us,scenario_023,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly rejected both the Saver's Credit and AOTC but used an estimated $15,400 standard deduction. Applying the actual $16,100 deduction reduces taxable income to $6,434.34 and tax to $643.43."
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model invoked an AOTC-related nonrefundable credit without any listed qualified education expenses. Enrollment and documentation establish personal eligibility but do not generate a credit when qualified expenses are zero.
+us,scenario_023,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model partially reduced the tax with an American Opportunity Credit despite $0 of listed qualified education expenses. No AOTC applies, so the $643.43 tentative tax is not reduced to $121."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated wage income below the standard deduction as dispositive and failed to carry the $8,000 taxable 403(b) distribution through the taxable-income calculation. Total AGI is $22,534.34, leaving $6,434.34 taxable after the standard deduction."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed the TCJA individual provisions expired for 2026 and substituted a smaller standard deduction plus a personal exemption. The applicable 2026 schedule uses a $16,100 single standard deduction, producing $6,434.34 of taxable income and $643.43 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"The model incorrectly awarded a $1,000 Saver's Credit after calculating tax under an inapplicable deduction-and-exemption regime. The $8,000 retirement distribution eliminates eligible Saver's Credit contributions, and the applicable standard deduction is $16,100."
+us,scenario_023,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, used an $8,500 standard deduction, and applied a 15% bracket above its estimated threshold. The applicable 2026 rules provide a $16,100 standard deduction, leaving all $6,434.34 of taxable income in the 10% bracket."
+us,scenario_023,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,089 employer-sponsored insurance premium from employment income, reducing AGI to $14,446. PolicyEngine's employment-income figure is reduced by the traditional 401(k) contribution but not by another employee income deduction for those employer premiums, yielding $22,534.34 of AGI."
 us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
 us,scenario_023,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
-us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model applied the nonrefundable AOTC despite zero listed qualified education expenses. Enrollment and 1098-T facts alone do not generate the credit, so the ordinary tax is not reduced to zero."
-us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model correctly recognized the IRA deduction but then applied an AOTC based only on education-status facts. With qualified education expenses fixed at zero, the AOTC is zero and cannot offset the $643.43 tax."
-us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model concluded that income was below the standard deduction by focusing on wages and failing to carry the $8,000 taxable 403(b) distribution through gross income. The resulting AGI exceeds the $16,100 deduction by $6,434.34, producing $643.43 of tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model awarded the nonrefundable AOTC despite the absence of listed qualified education expenses. The qualifying student facts do not create an AOTC with a $0 expense base, so no credit offsets the tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model computed a $1,000 Saver's Credit without subtracting the $8,000 taxable 403(b) distribution from eligible retirement contributions. That distribution exceeds the contributions, leaving a zero credit base and no reduction of the $643.43 tax."
+us,scenario_023,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model used an AOTC to eliminate the tax based only on enrollment and student-status facts. No qualified education expenses are listed, so the AOTC expense base and credit are both zero; it also used an estimated standard deduction instead of $16,100."
+us,scenario_023,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that income was fully absorbed by the standard deduction, overlooking the effect of the $8,000 taxable 403(b) distribution in addition to post-deferral wages. The resulting AGI is $22,534.34 and taxable income is $6,434.34, not zero."
 us,scenario_023,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model explicitly found that qualified tuition expenses were zero and that the AOTC therefore computed to zero, then overrode that result by inventing default education expenses. PolicyEngine does not impute qualified expenses from the enrollment flags, so the refundable AOTC is zero."
 us,scenario_023,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model awarded an AOTC solely from enrollment and documentation flags despite zero listed qualified education expenses. It also treated the entire $2,500 maximum AOTC as refundable, although only 40%, capped at $1,000, can be refundable even when sufficient expenses exist."
 us,scenario_023,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented roughly $400 of refundable AOTC without any qualified education expenses and added a residual EITC despite recognizing that the childless credit phases out at this AGI. Zero expenses produce zero AOTC, and AGI including the taxable 403(b) distribution eliminates the EITC."
@@ -1199,174 +1257,184 @@ us,scenario_023,free_school_meals_eligible,claude-opus-4.8,llm_error,categorical
 us,scenario_023,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, violating the required submission contract."
 us,scenario_023,head_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly treated California CHIP as covering disabled adults up to an extended 266% FPL threshold. It failed to apply the categorical exclusion for a person already Medicaid-eligible through the WORKING_DISABLED_BUY_IN pathway, which makes this head ineligible for CHIP regardless of the CHIP income thresholds."
 us,scenario_023,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable output for head_chip_eligible. The required result was 0 because Medicaid eligibility under WORKING_DISABLED_BUY_IN categorically excludes the head from CHIP.
-us,scenario_023,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model treated the 138% FPL Medi-Cal adult-expansion ceiling as the only applicable pathway. It omitted the WORKING_DISABLED_BUY_IN category established by the head's disability and employment.
-us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model compared MAGI solely with the 138% FPL expansion threshold and stopped. The head instead qualifies under the separate WORKING_DISABLED_BUY_IN category.
-us,scenario_023,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,The model tested adult expansion and SSI-linked Medicaid but omitted the working-disabled buy-in pathway. Failure of the expansion and SSI-linked tests does not negate eligibility under WORKING_DISABLED_BUY_IN.
-us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model's unsupported ineligibility conclusion omitted the WORKING_DISABLED_BUY_IN category. The head's disability and employment place the person in that separate Medicaid eligibility pathway.
-us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model disregarded the expressly listed disability and work facts as insufficient Medicaid-category details. Those facts establish the WORKING_DISABLED_BUY_IN category used for eligibility.
-us,scenario_023,head_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model applied the ordinary adult Medi-Cal income limit to the head's wages and 403(b) distribution as though it exhausted eligibility. It failed to evaluate the separate WORKING_DISABLED_BUY_IN pathway.
-us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly stated that no separate Medicaid category was established. The listed disability and employment establish the WORKING_DISABLED_BUY_IN category.
-us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model considered only adult expansion and SSI-linked eligibility. It omitted WORKING_DISABLED_BUY_IN, which is distinct from SSI-linked Medicaid and applies to this employed disabled head."
-us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model denied eligibility solely because MAGI exceeded the adult-expansion limit. It never applied the separate WORKING_DISABLED_BUY_IN category.
-us,scenario_023,head_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model tested the expansion and aged-blind-disabled income pathways but failed to test the working-disabled buy-in pathway. The head qualifies through WORKING_DISABLED_BUY_IN regardless of failing those two alternatives.
-us,scenario_023,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,The model limited its analysis to expansion Medicaid and a generic non-MAGI aged-blind-disabled test. It omitted the distinct WORKING_DISABLED_BUY_IN category that makes the employed disabled head eligible.
-us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no head_medicaid_eligible value or explanation, violating the required output contract."
-us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no head_medicaid_eligible value or explanation, violating the required output contract."
-us,scenario_023,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated expansion coverage and SSI receipt as the only routes to Medicaid. It omitted WORKING_DISABLED_BUY_IN, which does not require current SSI receipt."
-us,scenario_023,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,The model applied a general California Medicaid income limit without evaluating categorical eligibility. The head qualifies through the separate WORKING_DISABLED_BUY_IN pathway.
+us,scenario_023,head_medicaid_eligible,claude-opus-5,llm_error,categorical_eligibility,False,The model treated California's 138% FPL adult-expansion limit as the only applicable pathway. It omitted the working-disabled buy-in category established by the head's disability and employment.
+us,scenario_023,head_medicaid_eligible,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model denied eligibility solely because MAGI exceeded 138% FPL. It failed to apply the separate working-disabled buy-in pathway, for which the MAGI expansion cutoff is not controlling."
+us,scenario_023,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,The model tested only adult-expansion and SSI-linked Medicaid eligibility. Its SSI countable-income analysis was irrelevant because the head qualifies through the separate working-disabled buy-in category without receiving SSI.
+us,scenario_023,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model concluded that the income-based rules denied coverage without applying the head's working-disabled buy-in category. Disability plus employment establishes the relevant categorical pathway.
+us,scenario_023,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model disregarded the explicitly listed disability and work facts as inadequate Medicaid details. Those facts place the head in the working-disabled buy-in category, so the model omitted the decisive eligibility pathway."
+us,scenario_023,head_medicaid_eligible,gpt-5.5,llm_error,categorical_eligibility,False,The model applied the ordinary adult Medi-Cal income limit to the head and stopped. It failed to evaluate the distinct working-disabled buy-in category triggered by disability and employment.
+us,scenario_023,head_medicaid_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model asserted that no separate qualifying category was established even though the prompt explicitly establishes both disability and paid work. Those facts support the working-disabled buy-in category independently of the adult-expansion MAGI threshold.
+us,scenario_023,head_medicaid_eligible,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model limited the alternatives to adult expansion and SSI-linked eligibility. It omitted the working-disabled buy-in pathway, which does not require SSI receipt and is the applicable category."
+us,scenario_023,head_medicaid_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated MAGI above the adult-expansion limit as dispositive. It never applied the separate working-disabled buy-in category established by the head's disability and employment.
+us,scenario_023,head_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,"The model checked adult expansion and the conventional aged, blind, or disabled income pathway but omitted the working-disabled buy-in category. The head qualifies through that employment-specific disability pathway rather than either category the model tested."
+us,scenario_023,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model rejected adult-expansion and non-MAGI ABD eligibility based on income, but those are not the decisive tests. It omitted the separate working-disabled buy-in pathway applicable to this disabled worker."
+us,scenario_023,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for head_medicaid_eligible, violating the required output contract."
+us,scenario_023,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for head_medicaid_eligible, violating the required output contract."
+us,scenario_023,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 138% FPL expansion limit and SSI receipt as the exhaustive Medicaid tests. It missed the working-disabled buy-in pathway, which applies without SSI receipt."
+us,scenario_023,head_medicaid_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model characterized the head as a standard expansion adult and then tested only MAGI expansion and the conventional aged, blind, or disabled pathway. It omitted the working-disabled buy-in category specifically available to a disabled person who is employed."
+us,scenario_023,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated income above California's ordinary Medicaid limit as dispositive. It failed to apply the working-disabled buy-in category, under which the head is eligible based on disability and employment."
 us,scenario_023,head_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the generic disability flag as a proxy for SSDI-based Medicare entitlement. Disability alone does not confer Medicare eligibility; the facts provide neither the required disability-benefit entitlement pathway nor end-stage renal disease, and age 28 does not satisfy age-based eligibility."
 us,scenario_023,head_medicare_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model incorrectly made disability by itself sufficient for under-65 Medicare eligibility. The head has no listed qualifying disability-benefit entitlement or end-stage renal disease and is under 65, so the correct eligibility value is 0."
 us,scenario_023,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable head_medicare_eligible output. The required derivation yields 0 because the 28-year-old head has no listed Medicare-qualifying entitlement or medical criterion.
 us,scenario_023,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_023,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_023,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model used a 1.13% California SDI rate and calculated only about $197 of state payroll tax. PolicyEngine's 2026 SDI calculation produces $226.75 on the $17,442.65 payroll-tax wage base."
-us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly excluded California SDI by asserting that California has no mandatory employee state payroll tax. It also submitted $1,335.85 despite its stated federal components totaling $1,334.38, but the decisive omission is the $226.75 SDI contribution."
-us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate, producing $209.32 instead of the $226.75 state payroll-tax component. Its federal FICA treatment was otherwise aligned with the required calculation."
-us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a 0.9% California SDI rate instead of the 2026 parameter that produces $226.75. It then submitted $1,467.32 even though its own listed components totaled $1,490.38."
-us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate rather than the rate producing $226.75 and did not preserve its own arithmetic, which it described as approximately $1,544 before submitting $1,546."
-us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model ultimately kept the full FICA wage base but applied a 1.1% California SDI rate, yielding $191.87. PolicyEngine's state payroll-tax component is $226.75."
-us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model falsely treated California SDI as repealed and included only Social Security and Medicare. California's mandatory employee SDI contribution adds $226.75 to the payroll-tax total.
-us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $8,089 employer-sponsored insurance premium from gross wages and applied FICA and SDI to only $9,354. PolicyEngine uses $17,442.65 of employment income as the payroll-tax gross-wage base and does not treat the listed premium as an employee pretax wage deduction."
-us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model applied a 1.1% California SDI rate and calculated $191.87 of state payroll tax. The 2026 PolicyEngine calculation produces $226.75.
-us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model included only Social Security and Medicare and omitted California's mandatory employee SDI contribution of $226.75. It also overstated Medicare slightly relative to the $252.92 reference component.
-us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model excluded employer-sponsored insurance premiums from taxable gross wages. PolicyEngine applies payroll taxes to $17,442.65 because the facts do not identify the $8,089 premium as an employee pretax payroll deduction."
-us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model reduced payroll-tax wages from $17,443 to $9,354 by subtracting the $8,089 employer-sponsored insurance premium. That premium is not an identified employee pretax deduction, and PolicyEngine uses the full $17,442.65 employment-income base while also producing $226.75 of SDI."
-us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a 1.1% California SDI rate, yielding $191.87 rather than the $226.75 state payroll-tax component. Social Security and Medicare were otherwise computed on the correct wage category."
-us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The $809.12 answer is consistent with applying FICA and SDI to a substantially reduced wage base rather than the $17,442.65 payroll-tax gross-wage base. The correct component derivation is $1,081.44 of Social Security, $252.92 of Medicare, and $226.75 of California SDI."
-us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,The model correctly kept retirement deferrals in the FICA base but applied a 1.2% California SDI rate. PolicyEngine calculates $226.75 of SDI rather than $209.32.
-us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model explicitly omitted mandatory California employee payroll tax. California SDI contributes $226.75, and its submitted number also does not equal 7.65% of the stated $17,443 wage base."
-us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscomputed 6.2% of $17,443 as about $1,070.10 instead of about $1,081.44 and omitted California SDI entirely. The required components are $1,081.44 of Social Security, $252.92 of Medicare, and $226.75 of SDI."
-us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model applied an estimated 1.2% California SDI rate, producing $209.32. PolicyEngine's 2026 state payroll-tax calculation produces $226.75."
-us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model calculated only the 7.65% federal FICA taxes and omitted California employee SDI. The omitted state payroll-tax component is $226.75.
-us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model included only employee Social Security and Medicare. California's mandatory employee SDI contribution adds $226.75 to those federal components.
-us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model correctly used full wages for FICA but used a projected 1.2% California SDI rate, yielding $209.32. PolicyEngine's SDI component is $226.75, and rounding the wrong subtotal to a whole dollar compounded the error."
-us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $2,778 traditional 401(k) contribution from wages for Social Security, Medicare, and SDI. Traditional 401(k) deferrals remain subject to employee payroll taxes, so PolicyEngine uses $17,442.65 as the base; the model also used the wrong SDI rate."
-us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model subtracted the $8,089 employer-sponsored insurance premium from wages and taxed only $9,354. PolicyEngine uses the full $17,442.65 employment-income base because no employee pretax payroll deduction of that amount is specified, and its California SDI component is $226.75."
-us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll-tax output or explanation, so it failed the required structured-output contract."
-us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no payroll-tax output or explanation, so it failed the required structured-output contract."
-us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model summed only Social Security and Medicare and omitted California employee SDI. The missing mandatory state payroll-tax component is $226.75.
-us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,The model's reasoning calculated a California SDI amount but its submitted value excluded SDI and retained only federal FICA. It also used a 1.1% SDI rate rather than the parameter producing the $226.75 state component.
-us,scenario_023,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $261.65 cannot result from applying the stated Social Security and Medicare rates to $17,442.65 of wages and also omits California SDI. The correct component calculation yields $1,081.44 of Social Security, $252.92 of Medicare, and $226.75 of SDI."
+us,scenario_023,payroll_tax,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 1.13% California SDI rate and calculated only $197.11 of state payroll tax. The traced California employee contribution is $226.75, so its federal components were essentially right but its state rate calculation understated the total."
+us,scenario_023,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that California has no mandatory employee state payroll tax and omitted the $226.75 SDI contribution. Its submitted $1,335.85 also does not equal its own stated federal calculation of $1,334.38."
+us,scenario_023,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate, producing $209.32 instead of the traced $226.75 California contribution. Social Security and Medicare were computed correctly apart from the small difference between rounded stated wages and the engine wage base."
+us,scenario_023,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a 0.9% California SDI rate rather than the state contribution in the trace. It then submitted $1,467.32 even though its own listed components total $1,490.38, adding a separate arithmetic-output inconsistency."
+us,scenario_023,payroll_tax,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 1.2% California SDI rate and therefore calculated only $209.32 of state payroll tax instead of $226.75. Its submitted $1,546 also fails to match its own component sum of approximately $1,543.71."
+us,scenario_023,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"After correctly rejecting an unsupported Section 125 wage reduction, the model applied a 1.1% California SDI rate and obtained $191.87. The traced state payroll-tax component is $226.75, which accounts for the material shortfall."
+us,scenario_023,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model explicitly omitted California SDI after incorrectly claiming that California has no mandatory employee-side state payroll tax. It submitted only Social Security and Medicare, leaving out the traced $226.75 California contribution."
+us,scenario_023,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the entire $8,089 employer-sponsored insurance premium from wages even though the stated gross wages remain the traced payroll-tax base and no employee Section 125 deduction was specified. It consequently understated Social Security, Medicare, and California payroll tax, while also using the wrong state rate."
+us,scenario_023,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly included all three tax categories but applied a 1.1% California SDI rate, yielding $191.87 rather than the traced $226.75 state contribution. That state-rate error produced its $1,526.26 total."
+us,scenario_023,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model included only Social Security and Medicare and omitted the mandatory $226.75 California SDI contribution. It also calculated Medicare as $252.97 instead of approximately $252.92.
+us,scenario_023,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model excluded employer-sponsored insurance premiums from the stated gross wages without an employee pre-tax payroll deduction fact. The trace uses $17,442.65 as the base for Social Security, Medicare, and California payroll tax, so reducing the base to roughly $9,354 caused the severe understatement."
+us,scenario_023,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model wrongly reduced payroll-tax wages from $17,443 to $9,354 by subtracting the $8,089 employer-sponsored insurance premium. The trace applies all payroll-tax components to $17,442.65, and the model also used an incorrect 1.1% California SDI rate."
+us,scenario_023,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a 1.1% California SDI rate and calculated $191.87 of state payroll tax. The traced California contribution is $226.75, so the incorrect state rate explains the shortfall."
+us,scenario_023,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The submitted $809.12 is far below even the $1,334.36 traced Social Security-plus-Medicare amount on $17,442.65 of wages. Its answer therefore applies the listed taxes to an improperly reduced wage base and also fails to include the full $226.75 California contribution."
+us,scenario_023,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model correctly kept retirement contributions in the FICA base but applied a 1.2% California SDI rate, producing $209.32 rather than the traced $226.75. That incorrect state component caused its understated total."
+us,scenario_023,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly stated that California has no mandatory state payroll tax and therefore omitted the $226.75 SDI contribution. Its submitted $1,375.966 also does not equal 7.65% of $17,443, so the final number is inconsistent with its stated method."
+us,scenario_023,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscomputed 6.2% of $17,443 as about $1,070.10 instead of $1,081.47 and then rounded the federal total down to $1,208. It also omitted the mandatory $226.75 California SDI contribution."
+us,scenario_023,payroll_tax,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated 1.2% California SDI rate, generating $209.32 rather than the traced $226.75 state payroll tax. Its federal FICA treatment was otherwise aligned with the traced wage base."
+us,scenario_023,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model stopped after calculating the 7.65% federal Social Security and Medicare taxes. It omitted the traced $226.75 mandatory California employee SDI contribution.
+us,scenario_023,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model calculated only employee Social Security and Medicare on wages. It omitted the $226.75 California SDI component required in the payroll-tax output.
+us,scenario_023,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model correctly treated retirement deferrals as FICA-taxable but used a projected 1.2% California SDI rate, producing $209.32 instead of $226.75. Rounding that already understated total to $1,544 did not correct the state component."
+us,scenario_023,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"The model applied a 1.2% California SDI rate and calculated $209.32 of state payroll tax. The trace contains a $226.75 California contribution, so its $1,544 rounded total remains understated."
+us,scenario_023,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $2,778 traditional 401(k) contribution from wages for Social Security, Medicare, and SDI. Traditional 401(k) deferrals do not reduce the traced payroll-tax gross-wage base, and the model additionally used the wrong California contribution rate."
+us,scenario_023,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model wrongly subtracted $8,089 of employer-sponsored insurance premiums from gross wages and applied the taxes to only $9,354. The trace uses $17,442.65 for all payroll-tax components, and the model also used an incorrect 1.2% California SDI rate."
+us,scenario_023,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output or explanation. This is a missing required-key failure rather than a substantive tax computation.
+us,scenario_023,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output or explanation. This is a missing required-key failure rather than a substantive tax computation.
+us,scenario_023,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model included only Social Security and Medicare and omitted California's mandatory employee SDI tax. The missing traced state component is $226.75.
+us,scenario_023,payroll_tax,ox-alpha,llm_error,thresholds_rates,False,"The model correctly kept retirement deferrals in the federal payroll-tax base but applied a 1.2% California SDI rate, yielding $209 rather than $226.75. Its rounding to $1,544 preserves that state-rate understatement."
+us,scenario_023,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The reasoning recognized that California SDI is mandatory and even added an estimated state component, but the submitted value reverted to federal Social Security and Medicare alone. The final output therefore omitted the entire traced $226.75 California contribution, while the reasoning also used an incorrect 1.1% state rate."
+us,scenario_023,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $261.65 does not implement the stated Social Security-and-Medicare method: those federal taxes alone total about $1,334.36 on the traced $17,442.65 wage base. The answer reflects a severely truncated or misapplied wage base and also omits the $226.75 California SDI contribution."
 us,scenario_023,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_023,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_023,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_023,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model treated the generic disability flag as sufficient for an uncapped SNAP excess-shelter deduction. The shelter cap leaves $876.51 of monthly net income rather than the model’s $345, and it also submitted $3,936 despite reasoning to much lower annual figures."
-us,scenario_023,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly denied eligibility under the ordinary gross-income test and treated high rent as disqualifying. TANF non-cash categorical eligibility applies, and the capped shelter deduction reduces net income to $876.51, producing a positive benefit."
-us,scenario_023,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model removed the excess-shelter cap solely because the head was described as disabled. SNAP’s special disability definition was not established, so the capped shelter deduction yields $876.51 monthly net income rather than the much lower figure used in its estimate."
-us,scenario_023,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,The model incorrectly applied an uncapped shelter deduction based on the generic disability input and also misstated the 20% earned-income deduction as $121 per month. The capped computation produces $876.51 in monthly net income and a far smaller allotment.
-us,scenario_023,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model denied SNAP using gross-income and shelter-cap reasoning without applying TANF non-cash categorical eligibility. After allowable deductions and the capped shelter deduction, net income is $876.51 and the household receives a positive allotment."
-us,scenario_023,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the generic disability flag as establishing an uncapped excess-shelter deduction, reducing net income to $328.30. The applicable shelter cap instead leaves $876.51 of monthly net income, and the benefit must also reflect the two maximum-allotment parameter periods in 2026."
-us,scenario_023,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,The model stopped at the ordinary 130% FPL gross-income test and failed to apply TANF non-cash categorical eligibility. The household passes the net-income test at $876.51 per month and receives SNAP.
-us,scenario_023,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model both overstated the medical deduction and applied an uncapped shelter deduction from the generic disability flag, driving net income to zero. The capped deduction calculation leaves $876.51 monthly net income, so the household does not receive the maximum allotment."
-us,scenario_023,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly added a utility allowance not supported by the listed facts and treated shelter expenses as uncapped. With unlisted expenses set to zero and the shelter cap applied, monthly net income is $876.51 rather than zero."
-us,scenario_023,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,The submitted near-maximum annual allotment omits the expected contribution from $876.51 of monthly net income. The correct computation subtracts 30% of that net income from each applicable monthly maximum.
-us,scenario_023,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the disability flag as sufficient to make the excess-shelter deduction uncapped and reduced net income to zero. The shelter cap applies, leaving $876.51 of monthly net income and a substantial expected contribution."
-us,scenario_023,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model estimated net income near $300 by using an uncapped shelter deduction for the disabled head. The shelter cap applies and leaves monthly net income of $876.51, sharply reducing the benefit."
-us,scenario_023,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,The model explicitly used uncapped shelter expenses based on the generic disability fact. Applying the shelter cap produces $876.51 in monthly net income and annual SNAP of $461.34.
-us,scenario_023,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced net income to zero through an uncapped disability shelter deduction. SNAP’s special disability status was not established, so the cap applies and the household owes a 30% contribution on $876.51 monthly net income."
-us,scenario_023,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated the full $14,836.80 excess shelter amount as deductible because of the generic disability flag. The shelter deduction is capped, producing $876.51 monthly net income rather than $3,489.60 annually."
-us,scenario_023,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model asserted that income and assets were too high without applying categorical eligibility or the SNAP deductions. Assets are only $130, and capped deductions produce net income of $876.51, which passes the net-income test."
-us,scenario_023,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly required separate evidence of benefit receipt or eligibility despite the prompt’s take-up assumption. TANF non-cash categorical eligibility and $876.51 monthly net income establish a positive SNAP benefit.
-us,scenario_023,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the excess-shelter deduction as uncapped and reduced net income to zero. The generic disability fact does not establish SNAP’s special disability status, so the capped deduction leaves $876.51 monthly net income."
-us,scenario_023,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model assumed that high rent and the disability flag eliminated all net income through an uncapped shelter deduction. The shelter cap applies, so the household receives only the maximum less 30% of $876.51 monthly net income."
-us,scenario_023,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"Although the model correctly recognized the student work-hours exception, it then applied an uncapped disability shelter deduction. The capped shelter calculation leaves $876.51 monthly net income, not the low net income underlying a $202 monthly allotment."
-us,scenario_023,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model concluded that earnings and the retirement distribution eliminated the benefit without carrying through categorical eligibility and allowable deductions. Those deductions reduce monthly net income to $876.51, yielding a positive allotment."
-us,scenario_023,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared both income and assets excessive; bank assets are only $130 and categorical eligibility applies. The net-income calculation after capped deductions is $876.51, below the applicable limit."
-us,scenario_023,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $14,837 excess shelter amount after treating the generic disability fact as an exemption from the shelter cap. Applying the cap leaves $876.51 of monthly net income and a much larger expected contribution."
-us,scenario_023,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from its income-before-shelter arithmetic and then applied an uncapped shelter deduction. Including that income and enforcing the shelter cap produces $876.51 monthly net income rather than zero."
-us,scenario_023,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model estimated monthly net income near $300 by applying an uncapped shelter deduction based on disability. The applicable cap leaves $876.51 monthly net income, so its expected contribution was substantially understated."
-us,scenario_023,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,The model treated the generic disability flag as sufficient for an uncapped excess-shelter deduction and therefore assigned the maximum allotment. The shelter cap applies and leaves $876.51 of monthly net income.
-us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output was missing."
-us,scenario_023,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model used an incorrect $22,535 gross-income figure and denied eligibility solely under the ordinary 130% FPL test. TANF non-cash categorical eligibility applies, and the net-income test is passed at $876.51 per month."
-us,scenario_023,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $8,000 taxable 403(b) distribution from gross income and never correctly computed the shelter deduction or 30% expected contribution. Including the distribution and capped deductions yields $876.51 monthly net income, not a maximum benefit."
-us,scenario_023,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model denied eligibility with a generic income-and-resources conclusion, overlooking $130 assets and TANF non-cash categorical eligibility. The household passes the net-income test at $876.51 per month and receives a positive benefit."
+us,scenario_023,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It treated the general disabled flag as establishing SNAP elderly/disabled status, applied a medical deduction and an uncapped excess-shelter deduction, and reduced net income to $345 monthly. The applicable deduction calculation yields net income around $876.51 and a much smaller annual allotment."
+us,scenario_023,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"It disqualified the household under the ordinary 130% FPL gross-income test and incorrectly portrayed high rent as adverse to eligibility. The household has TANF non-cash categorical eligibility, passes the applicable net-income and asset tests, and receives a positive benefit."
+us,scenario_023,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,It converted the general disability input into SNAP elderly/disabled status and deducted both excess medical costs and the full uncapped shelter excess. SNAP net income is instead about $876.51 after the applicable capped deduction calculation.
+us,scenario_023,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It miscomputed the earned-income deduction as $121 monthly instead of 20% of monthly wages and then applied SNAP medical and uncapped shelter deductions based solely on the general disability flag. Its final $2,549 also contradicts its own stated $149 monthly calculation, which annualizes to $1,788."
+us,scenario_023,snap,claude-opus-5,llm_error,categorical_eligibility,False,It concluded that gross or net income eliminated the benefit while applying the ordinary income-limit framework. TANF non-cash categorical eligibility and the applicable capped deduction calculation leave net income around $876.51 and produce a positive allotment.
+us,scenario_023,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated the disabled flag as authorization for the SNAP medical deduction and removal of the excess-shelter cap, reducing monthly net income to $328.30. The applicable deductions leave about $876.51 of net income, and the 2026 benefit must also reflect the parameter change across months rather than one estimated monthly amount multiplied by 12."
+us,scenario_023,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,It denied SNAP under the ordinary 130% FPL gross-income test and never applied TANF non-cash categorical eligibility. The household passes the applicable net-income and asset tests and receives $461.34.
+us,scenario_023,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It used a $747.42 medical deduction and an uncapped $1,560.94 shelter deduction to force net income to zero. Those elderly/disabled SNAP deductions do not follow from the general disability flag, and the applicable calculation leaves net income around $876.51."
+us,scenario_023,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It added a utility allowance not listed in the facts and applied both a medical deduction and uncapped shelter treatment from the general disability flag. The prompt sets unlisted expenses to zero, and the applicable deductions leave positive net income around $876.51."
+us,scenario_023,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its $3,492 answer is a full-year maximum allotment shortcut. PolicyEngine derives positive net income around $876.51 and subtracts the 30% expected contribution in each month, yielding only $461.34 annually."
+us,scenario_023,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,It treated the general disability flag as establishing uncapped SNAP shelter and medical deductions and therefore reduced net income to zero. The applicable deduction calculation leaves monthly net income around $876.51.
+us,scenario_023,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It estimated net income near $300 by applying uncapped shelter and medical deductions to the disabled head. The applicable SNAP deduction rules leave net income around $876.51, so the expected contribution is substantially larger."
+us,scenario_023,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It based the $1,980 estimate on uncapped shelter expenses for a disabled individual. The household does not obtain that SNAP deduction treatment from the general disability flag, and its monthly net income is about $876.51."
+us,scenario_023,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,It removed the shelter cap based on the general disability flag and reduced net income to zero. The applicable capped deduction calculation produces net income around $876.51 rather than eligibility for the maximum allotment.
+us,scenario_023,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"It deducted $1,180 of medical expenses and $14,836.80 of uncapped shelter costs by treating the general disability flag as SNAP elderly/disabled status. Those deductions overstate the allowable reduction, leaving its annual net income far below PolicyEngine's approximately $876.51 monthly amount."
+us,scenario_023,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It inferred zero from a generic assessment that income and assets were too high without applying the stated $130 asset amount, TANF non-cash categorical eligibility, or the net-income calculation. Those rules produce eligibility and a positive $461.34 benefit."
+us,scenario_023,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,It required separate evidence of SNAP receipt or eligibility even though the prompt directs assumed take-up and supplies the facts needed to calculate eligibility. TANF non-cash categorical eligibility and the passing net-income and asset tests produce a positive benefit.
+us,scenario_023,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied medical and uncapped excess-shelter deductions based solely on the general disability flag and forced countable net income to zero. The applicable deductions leave net income around $876.51, requiring a 30% expected contribution."
+us,scenario_023,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,It assumed that disability automatically permitted medical and uncapped shelter deductions and awarded the maximum allotment. The SNAP computation instead leaves about $876.51 in monthly net income after applicable deductions.
+us,scenario_023,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It correctly recognized the student work-hours pathway but overstated medical and shelter deductions under disability treatment, producing roughly $202 per month. The applicable deduction calculation leaves net income around $876.51 and benefits of only $35.20 to $48.18 per month."
+us,scenario_023,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"It concluded that earnings and the retirement distribution exhausted the benefit without applying the full capped deduction and categorical-eligibility computation. The resulting net income is about $876.51, low enough for a positive allotment."
+us,scenario_023,snap,grok-4.3,llm_error,categorical_eligibility,False,It asserted that income and assets exceeded the limits despite bank assets of only $130 and TANF non-cash categorical eligibility. The applicable net-income test is also satisfied at about $876.51 monthly.
+us,scenario_023,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"It treated the general disability flag as exemption from ordinary SNAP deduction limits and deducted medical expenses plus $14,837 of uncapped shelter costs. The applicable capped calculation leaves monthly net income around $876.51, not annual net income of only $3,490."
+us,scenario_023,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"It used medical and uncapped shelter deductions to reduce net SNAP income to zero. The general disabled flag does not itself establish that SNAP deduction pathway, and the applicable calculation produces about $876.51 of monthly net income."
+us,scenario_023,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It omitted the $8,000 taxable 403(b) distribution from its post-deduction income arithmetic and then used an uncapped shelter deduction to force net income to zero. Both errors improperly produce a maximum allotment instead of the small residual benefit."
+us,scenario_023,snap,inkling,llm_error,taxable_income_or_deductions,False,"It estimated monthly net income near $300 using medical and uncapped shelter deductions tied to the general disability flag. The applicable SNAP calculation leaves about $876.51 monthly, making the expected contribution much higher."
+us,scenario_023,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It treated the general disability input as establishing exemption from the shelter cap and eligibility for a medical deduction, reducing net income to zero. The applicable deductions instead leave monthly net income around $876.51."
+us,scenario_023,snap,kimi-k3,parse_contract_failure,missing_output,False,"It supplied no SNAP value or explanation, so the required output was missing."
+us,scenario_023,snap,minimax-m3,llm_error,categorical_eligibility,False,"It used the ordinary 130% FPL gross-income test and also misstated gross income as $22,535 instead of $25,443. TANF non-cash categorical eligibility bypasses that shortcut, after which the household passes the applicable net-income test."
+us,scenario_023,snap,ox-alpha,llm_error,thresholds_rates,False,"It identified the capped shelter treatment and reached a benefit close to the target, but used estimated single-period parameters and annualized roughly $34 for all 12 months. The 2026 calculation uses monthly net income around $876.51 and aggregates $35.20 monthly benefits with later $48.18 benefits after the maximum-allotment and deduction parameters change."
+us,scenario_023,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the $8,000 taxable 403(b) distribution from gross income and then awarded the full maximum despite calculating positive net income near $944. A positive net-income amount requires subtracting 30% as the expected contribution, and the complete income calculation yields about $876.51 monthly."
+us,scenario_023,snap,qwen3.8-max,llm_error,categorical_eligibility,False,It declared income and resources too high without applying TANF non-cash categorical eligibility or recognizing that $130 of bank assets passes the asset test. The applicable net-income calculation also passes and produces a positive benefit.
 us,scenario_023,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the narrative phrase “is disabled” as establishing SSI disability eligibility, despite the prompt directing that unlisted program-status inputs are false. The head fails the SSI aged, blind, or disabled gate, so applying earned-income exclusions, counting the 403(b) distribution, and subtracting income from the federal-plus-California payment standard was inapplicable."
 us,scenario_023,ssi,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model rounded the tentative tax and credits until it treated $216.47 of credits as fully offsetting $223.27 of tax. Applying the exact 2026 bracket parameters leaves $6.80.
-us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,The model incorrectly treated CalEITC as a nonrefundable credit reducing this output; CalEITC belongs in California refundable credits. It also omitted the taxable 403(b) distribution and retirement adjustments from its AGI calculation.
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invoked unspecified disability considerations to eliminate the remaining liability. The applicable credits total $216.47, consisting of $156.47 in exemption credits and the $60 renter credit, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly identified that the personal exemption alone does not eliminate the tax, then forced the residual liability to zero through unsupported rounding. Exact tentative tax of $223.27 less $216.47 of nonrefundable credits equals $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,The model asserted that roughly $150 of exemption credit eliminated roughly $200 of tax without accounting for the residual. The exact computation applies $216.47 of total credits against $223.27 of tax and leaves $6.80.
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented an additional California disability exemption credit equal to the personal exemption credit. PolicyEngine applies $156.47 in exemption credits and the $60 renter credit, not two separate approximately $144 exemption credits, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model relied on unspecified disability-related credits and then claimed the personal exemption exceeded tentative tax despite estimating tax at $250–$300 and the exemption near $149. The applicable exemption and renter credits total $216.47 against $223.27 of tax, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model computed CA AGI as $14,446 by dropping the taxable $8,000 403(b) distribution. Including that distribution yields $22,534.34 of AGI, $16,828.34 of taxable income, and $6.80 after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used approximate standard-deduction, bracket, and exemption amounts, producing an overstated $15 residual. The exact values produce $223.27 of tentative tax and $216.47 of credits, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that California's standard deduction exceeds AGI. AGI is $22,534.34, the standard deduction is only $5,706, and taxable income is $16,828.34."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly treated the personal exemption credit as sufficient to eliminate the tax. Tentative tax is $223.27, and even after $156.47 of exemption credits plus the $60 renter credit, $6.80 remains."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated taxable income as $8,846 by omitting taxable income or applying unsupported deductions. California taxable income is $16,828.34, producing $223.27 of tentative tax before $216.47 of credits."
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,The answer is consistent with stopping near tentative California tax and failing to subtract the applicable nonrefundable exemption and renter credits. Those credits total $216.47 and reduce $223.27 to $6.80.
-us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model rounded the residual after the standard deduction and personal credits down to zero. Exact tentative tax of $223.27 less $216.47 in applicable nonrefundable credits leaves $6.80.
-us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model expressly applied no nonrefundable state credits and also treated the personal exemption as a deduction from taxable income. California instead applies $156.47 in exemption credits and a $60 renter credit against $223.27 of tentative tax, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly said the standard deduction and credits wipe out the liability. The $5,706 deduction leaves $16,828.34 taxable, and $216.47 of credits against $223.27 of tax leaves $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly invoked nonrefundable California amounts associated with the American Opportunity Credit, which is a federal credit rather than a California credit in this calculation. The state computation uses $156.47 of exemption credits and the $60 renter credit, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model invented a 2.5% California additional tax on the $8,000 403(b) distribution and reported that $200 penalty as this output. The distribution enters AGI, while regular California tax after the applicable nonrefundable credits is $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model treated the personal exemption and renter credit as fully offsetting tentative tax. They total $216.47, which is $6.80 less than the $223.27 tentative liability."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used projected and rounded California parameters, understating the residual as $4. Exact 2026 taxable income is $16,828.34, tentative tax is $223.27, and credits are $216.47, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model incorrectly concluded that the exemption and renter credits fully offset the low-income tax. The credits total $216.47 against $223.27 of tentative tax, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction and low income as producing zero tax before properly calculating taxable income. The $5,706 standard deduction leaves $16,828.34 taxable, with $6.80 remaining after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model approximated the deduction, bracket cutoffs, and exemption credit and therefore overstated the residual as $9. Exact tentative tax is $223.27 and exact nonrefundable credits are $216.47, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model subtracted the $60 renter credit but omitted the $156.47 exemption credits. Applying both credit components to $223.27 of tentative tax leaves $6.80.
-us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated AGI as roughly $14,446 by dropping the taxable $8,000 403(b) distribution. Including it produces $22,534.34 of AGI and $16,828.34 of taxable income, leaving $6.80 after credits."
-us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
-us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
-us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly added back the traditional 401(k) and IRA deductions, inflating CA AGI to $25,443, and then applied no nonrefundable credits. AGI is $22,534.34, and $216.47 of exemption and renter credits reduce $223.27 of tax to $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model failed to deduct the traditional retirement contributions from AGI and invented a second exemption credit solely for disability, then used those offsetting errors to reach zero. The correct calculation uses $22,534.34 of AGI and $156.47 of exemption credits plus the $60 renter credit, leaving $6.80."
-us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that the standard deduction and nonrefundable credits eliminate the tax without computing the residual. Exact tax is $223.27 and applicable credits are $216.47, so $6.80 remains."
-us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model reduced wages by the desired traditional 401(k) contribution and then estimated the phaseout instead of applying the California schedule to adjusted earnings of $17,442.65. The applicable CalEITC computation yields $148.31, not $402."
-us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of dependents as disqualifying. California permits an otherwise eligible age-28 filer without qualifying children to receive CalEITC, and the schedule produces $148.31."
-us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model never consistently applied the childless CalEITC schedule, first deriving roughly $44 and then submitting $358 based on unspecified credit interactions it expressly recognized were not state credits. Adjusted earnings of $17,442.65 produce $148.31 under the California formula."
-us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified CalEITC as the sole applicable refundable credit but assigned a combined amount of $1,208 without applying the childless phaseout schedule. The schedule applied to $17,442.65 of adjusted earnings yields $148.31."
-us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an incorrect childless CalEITC phaseout endpoint and declared the credit exhausted. At adjusted earnings of $17,442.65 the California credit remains positive and equals $148.31."
-us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified the $8,000 taxable 403(b) distribution as disqualifying investment income for CalEITC. The filer remains eligible, and applying California's formula to adjusted earnings of $17,442.65 yields $148.31."
-us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model identified the correct credit and eligibility but substituted an unsupported approximate amount for the California schedule calculation. The childless CalEITC at adjusted earnings of $17,442.65 is $148.31, not $285."
-us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly computed CalEITC as 85% of a federal EITC amount. California uses its own credit schedule, which produces $148.31 from adjusted earnings of $17,442.65."
-us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model used a constructed $22,535 AGI as the phaseout base and estimated the result. PolicyEngine's California calculation uses adjusted earnings of $17,442.65, for which the CalEITC schedule yields $148.31."
-us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly concluded that income eliminated all California refundable credits. The filer qualifies for childless CalEITC, and adjusted earnings of $17,442.65 generate $148.31."
-us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model identified CalEITC but approximated its amount instead of applying the California schedule. At adjusted earnings of $17,442.65, the credit is $148.31 rather than $180."
-us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied an incorrect CalEITC income limit and treated the filer as over the threshold. Adjusted earnings of $17,442.65 remain within the childless credit schedule and yield $148.31."
-us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model recognized CalEITC eligibility but did not calculate the applicable schedule amount. Applying it to adjusted earnings of $17,442.65 gives $148.31, not $195."
-us,scenario_023,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly used $22,535 of AGI and an asserted lower phaseout endpoint to eliminate the credit. The California computation uses adjusted earnings of $17,442.65 and returns $148.31."
-us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model overlooked the childless CalEITC eligibility pathway. This age-28 California worker qualifies, and adjusted earnings of $17,442.65 yield a refundable credit of $148.31."
-us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model overlooked the childless CalEITC available from the listed wage and age facts. The California schedule produces $148.31 from adjusted earnings of $17,442.65."
-us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly isolated CalEITC but used a rough phaseout estimate rather than the California formula. The schedule amount at adjusted earnings of $17,442.65 is $148.31, not $114."
-us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model approximated the childless CalEITC instead of applying its exact schedule. Adjusted earnings of $17,442.65 produce $148.31 rather than $194."
-us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model identified the applicable CalEITC but estimated the phaseout amount. The California schedule applied to adjusted earnings of $17,442.65 yields $148.31, not $160."
-us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model failed to apply the childless CalEITC pathway to the listed wage and age facts. The filer is eligible, and adjusted earnings of $17,442.65 generate $148.31."
-us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly stated that no refundable state credit applied. California's childless CalEITC applies to this filer and equals $148.31 at adjusted earnings of $17,442.65."
-us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly made California EITC eligibility depend on qualifying for federal EITC and used the federal no-child phaseout to deny the state credit. California's separate CalEITC rules yield $148.31.
-us,scenario_023,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model constructed a CalEITC estimate from a 2024 maximum and an assumed 3.25% phaseout rate applied to $22,535 of AGI. The 2026 California schedule uses adjusted earnings of $17,442.65 and yields $148.31."
-us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model identified the correct refundable credit but replaced the schedule calculation with a broad estimate. CalEITC at adjusted earnings of $17,442.65 is $148.31, not $300."
-us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits. The required output was the $148.31 CalEITC amount.
-us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits. The required output was the $148.31 CalEITC amount.
-us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly asserted that CalEITC is nonrefundable for filers without children. CalEITC is refundable for eligible childless filers, and this household receives $148.31."
-us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,The model overestimated CalEITC and incorrectly added California's renter's credit as refundable even though that credit is nonrefundable. The sole refundable state credit is CalEITC of $148.31.
-us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model overlooked the childless CalEITC available to this age-28 California worker. Applying the California schedule to adjusted earnings of $17,442.65 yields $148.31."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model rounded the tentative tax and credits until it treated the remaining liability as effectively zero. Applying the exact $223.27 tentative tax and $216.47 of nonrefundable credits leaves $6.80.
+us,scenario_023,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated CalEITC as a nonrefundable credit that offsets this output and omitted the taxable $8,000 distribution and retirement adjustments from its taxable-income calculation. CalEITC belongs in refundable state credits; the applicable nonrefundable exemption and renter credits leave $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model understated tentative tax as about $170 and invoked unspecified disability considerations to eliminate it. The brackets produce $223.27, and only $216.47 of applicable nonrefundable exemption and renter credits reduce this output, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized that the personal exemption alone does not eliminate the tax, then nevertheless rounded the residual to zero without computing the renter credit and exact balance. The $156.47 exemption credit plus $60 renter credit reduce $223.27 to $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model asserted that a roughly $150 personal exemption credit fully offsets roughly $200 of tax, omitting the exact renter-credit calculation and residual liability. The applicable credits total $216.47 against $223.27 of tax, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented a second California exemption credit solely because the taxpayer is disabled. The trace applies $156.47 of exemptions and the $60 renter credit, not two personal exemption credits, so $223.27 falls to $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invoked undefined disability-related credits and claimed the personal exemption exceeded tentative tax despite its own $250–$300 estimate. California applies $156.47 of exemptions and a $60 renter credit here, leaving $6.80 from $223.27."
+us,scenario_023,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced California AGI to $14,446, effectively excluding the taxable $8,000 403(b) distribution after the retirement deductions. Correct AGI is $22,534.34 and taxable income is $16,828.34, producing $6.80 after credits."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used approximate projected deduction, bracket, and exemption values, producing about $230 of tax and a $15 residual. The exact parameters yield $223.27 of tax and $216.47 of credits, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that California's standard deduction exceeds AGI. AGI is $22,534.34 and the standard deduction is only $5,706, leaving $16,828.34 taxable and $6.80 after credits."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model incorrectly treated the personal exemption credit as sufficient to eliminate the tax. Tentative tax is $223.27, while the exemption and renter credits total $216.47, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated taxable income as $8,846 instead of $16,828.34, excluding a substantial portion of taxable income. Correct tentative tax is $223.27, and the exemption and renter credits leave $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The $218 answer is consistent with estimating tentative California tax while failing to subtract the applicable exemption and renter credits. Exact tentative tax is $223.27 and nonrefundable credits total $216.47, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model incorrectly concluded that the standard deduction and personal exemption credits eliminate all liability. They produce $223.27 of tentative tax and, together with the renter credit, only $216.47 of offsets, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model treated a personal exemption as a deduction in computing taxable income and then stated that no nonrefundable credits apply. California instead allows the $5,706 standard deduction and applies $156.47 of exemption credits plus the $60 renter credit, reducing $223.27 to $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model lumped the standard deduction, personal exemption, and other credits together and incorrectly concluded they wipe out the tax. The deduction leaves $16,828.34 taxable, and the $216.47 of credits leave $6.80 against $223.27 of tax."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly applied nonrefundable amounts associated with the federal American Opportunity Credit to California tax and failed to apply the actual California credit total. The applicable California exemption and renter credits total $216.47, reducing $223.27 to $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model replaced the regular-tax residual with a 2.5% California early-distribution additional tax on the entire $8,000 distribution. This output's traced computation is regular tentative tax of $223.27 less $216.47 in nonrefundable credits, yielding $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model incorrectly treated the personal exemption and renter credit as fully offsetting the tentative tax. Those credits total $216.47 against $223.27, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model followed the correct calculation structure but used projected parameters that left $4. Exact taxable income is $16,828.34, tentative tax is $223.27, and credits are $216.47, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model incorrectly concluded that the personal-exemption and renter credits fully offset the tax. Their exact total is $216.47, which leaves $6.80 of the $223.27 tentative liability."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction and low income produce zero tax before addressing the taxable $8,000 distribution and remaining taxable income. The $5,706 deduction leaves $16,828.34 taxable, and tax after applicable credits is $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used approximate projected deduction, bracket, and exemption parameters and obtained a $9 residual. Exact tentative tax is $223.27 and exact nonrefundable credits are $216.47, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted an itemized medical deduction of about $7,499 despite only $1,300 of listed non-premium medical expenses and the trace's use of the standard deduction. California uses the $5,706 standard deduction here, producing $16,828.34 taxable income and $6.80 after credits."
+us,scenario_023,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model applied the $60 renter credit but omitted the $156.47 exemption credit. Subtracting both credits from the exact $223.27 tentative tax leaves $6.80.
+us,scenario_023,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated AGI as about $14,446, effectively dropping the taxable $8,000 403(b) distribution after retirement deductions. Correct AGI is $22,534.34 and taxable income is $16,828.34, leaving $6.80 after credits."
+us,scenario_023,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract."
+us,scenario_023,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract."
+us,scenario_023,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model wrongly added back the deductible traditional 401(k) and IRA contributions, raising California AGI from $22,534.34 to $25,443. It also omitted $216.47 of exemption and renter credits; the correct sequence leaves $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used approximate deduction, bracket, and exemption values, producing $229 of tax and a $20 residual. Exact tentative tax is $223.27 and exact credits total $216.47, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model repeatedly used the wrong filing-status deduction and AGI, then invented a second $144 exemption credit for disability. The correct single-filer calculation uses $22,534.34 of AGI, a $5,706 standard deduction, and only $156.47 of exemptions plus the $60 renter credit, leaving $6.80."
+us,scenario_023,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly asserted that the standard deduction and nonrefundable credits eliminate California tax at this income. They yield $223.27 of tentative tax and $216.47 of credits, leaving $6.80."
+us,scenario_023,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly reduced CalEITC earnings by the desired traditional 401(k) contribution and then estimated the phaseout. The California formula uses adjusted earnings of $17,442.65 here and yields $148.31, not $402."
+us,scenario_023,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of dependents as disqualifying. California permits a qualifying worker without children to receive CalEITC, and the credit at adjusted earnings of $17,442.65 is $148.31."
+us,scenario_023,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model recognized that only CalEITC applies but never consistently applied the childless-filer formula, first deriving roughly $44 and then submitting $358 without a supporting computation. The 2026 formula at $17,442.65 of adjusted earnings yields $148.31."
+us,scenario_023,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified CalEITC as the sole applicable credit but assigned $1,208 despite describing the childless credit as small and phased out. Applying the childless-filer schedule to $17,442.65 yields $148.31."
+us,scenario_023,state_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used a phaseout ceiling below the applicable California limit and eliminated CalEITC. Adjusted earnings of $17,442.65 remain eligible and produce $148.31."
+us,scenario_023,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified the $8,000 taxable 403(b) distribution as disqualifying CalEITC investment income. That distribution does not eliminate the credit under the applicable California computation, which awards $148.31 based on $17,442.65 of adjusted earnings."
+us,scenario_023,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model identified the correct credit and eligibility pathway but estimated the childless CalEITC instead of applying the 2026 formula. The schedule at adjusted earnings of $17,442.65 produces $148.31, not $285."
+us,scenario_023,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly calculated CalEITC as a flat 85% of federal EITC. California uses its own CalEITC schedule, which produces $148.31 from adjusted earnings of $17,442.65."
+us,scenario_023,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model phased CalEITC using its constructed $22,535 AGI instead of applying the California calculation to adjusted earnings of $17,442.65. The applicable formula yields $148.31."
+us,scenario_023,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the California Earned Income Tax Credit. A childless filer with $17,442.65 of adjusted earnings remains eligible and receives $148.31."
+us,scenario_023,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model recognized CalEITC eligibility but approximated the amount without applying the exact 2026 schedule. Adjusted earnings of $17,442.65 produce $148.31, not $180."
+us,scenario_023,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly placed the head's income above the CalEITC eligibility limit. The childless-filer schedule still applies at adjusted earnings of $17,442.65 and yields $148.31."
+us,scenario_023,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model identified CalEITC but used the wrong phaseout amount. California's 2026 formula at $17,442.65 of adjusted earnings gives $148.31, not $195."
+us,scenario_023,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model eliminated CalEITC by applying the phaseout to a $22,535 AGI measure. The applicable computation uses adjusted earnings of $17,442.65 and returns $148.31."
+us,scenario_023,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked the California Earned Income Tax Credit. The supplied wages and childless-filer status generate $148.31 of CalEITC under the 2026 formula.
+us,scenario_023,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model omitted CalEITC despite the head satisfying its applicable worker-without-children pathway. Adjusted earnings of $17,442.65 yield a refundable credit of $148.31."
+us,scenario_023,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model selected the correct credit but estimated the phaseout at $114. Applying the 2026 childless CalEITC formula to $17,442.65 produces $148.31."
+us,scenario_023,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly limited the result to CalEITC but approximated its amount with the wrong phaseout parameters. The exact credit at adjusted earnings of $17,442.65 is $148.31."
+us,scenario_023,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model recognized the childless CalEITC pathway but estimated $160 rather than applying the exact schedule. The 2026 computation at $17,442.65 yields $148.31."
+us,scenario_023,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model failed to generate the California Earned Income Tax Credit. The head qualifies through the worker-without-children pathway, producing $148.31 at adjusted earnings of $17,442.65."
+us,scenario_023,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable California credit applied. CalEITC covers eligible filers without qualifying children and supplies $148.31 here.
+us,scenario_023,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly conditioned CalEITC eligibility on qualifying for federal EITC. California's credit has its own eligibility rules and schedule, under which $17,442.65 of adjusted earnings yields $148.31."
+us,scenario_023,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"The model incorrectly inferred that zero federal EITC forces CalEITC to zero. California independently computes CalEITC, and its formula awards $148.31 here."
+us,scenario_023,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used estimated 2024-derived maximum, phaseout-start, and phaseout-rate parameters and applied them to its $22,535 AGI. The applicable 2026 California schedule uses adjusted earnings of $17,442.65 and produces $148.31."
+us,scenario_023,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model identified CalEITC as the sole applicable credit but replaced the statutory calculation with a rough $300 estimate. The exact 2026 amount at $17,442.65 of adjusted earnings is $148.31."
+us,scenario_023,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable state_refundable_credits value or explanation.
+us,scenario_023,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable state_refundable_credits value or explanation.
+us,scenario_023,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly stated that CalEITC is nonrefundable for filers without children. CalEITC is refundable on the childless-filer pathway, and this household receives $148.31."
+us,scenario_023,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model both reduced earnings to $14,665 using the desired traditional 401(k) contribution and applied an erroneously low childless phaseout ceiling. The correct adjusted earnings are $17,442.65 and remain within the schedule, yielding $148.31."
+us,scenario_023,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model overestimated CalEITC and then incorrectly added California's renter's credit to refundable credits even though that credit is nonrefundable. The sole refundable amount is the $148.31 CalEITC calculated from $17,442.65 of adjusted earnings."
+us,scenario_023,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted the California Earned Income Tax Credit. The childless head qualifies at adjusted earnings of $17,442.65, producing $148.31."
 us,scenario_023,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_023,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning computed $7,019 using its estimated parameters, then submitted $3,325 without any computation supporting that value. This is an internal final-answer substitution error; the traced inputs produce $62,725.30 of taxable income and $7,031.04 of tax."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $29,900 standard deduction instead of $32,200 and then misapplied the married-filing-jointly rate schedule: even its stated $65,025 taxable income does not generate $8,640 under the applicable 10% and 12% brackets. The correct taxable income is $62,725.30 and the bracket calculation yields $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model replaced the exact 2026 parameters with rounded estimates of a $32,300 deduction and a $24,000 first-bracket ceiling. PolicyEngine uses the $32,200 standard deduction and the applicable 2026 bracket threshold, producing $7,031.04 rather than $7,035."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly derived roughly $7,031 from the correct $32,200 standard deduction and bracket structure, then changed the submitted result to $6,934 without a supporting adjustment. No credit or modification reduces the calculated tax to that number."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction instead of the 2026 married-filing-jointly amount of $32,200, leaving taxable income $2,200 too high. Its use of older bracket thresholds compounded the error, while the correct $62,725.30 taxable income yields $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived taxable income of about $62,725 and tax of about $7,031, then submitted $4,700 despite identifying no nonrefundable credit or other reduction. The submitted value contradicts its own completed computation."
-us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $30,000 standard deduction and 2025 bracket threshold rather than the 2026 $32,200 deduction and applicable 2026 schedule. That overstated taxable income as $64,925 instead of $62,725.30 and produced $7,314."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the listed employer-sponsored insurance premium from wages, invented mortgage interest and state taxes from balances, and added personal exemptions. Those inputs do not reduce AGI or taxable income here; AGI is $94,925.30 and the $32,200 standard deduction controls."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $22,400 married-filing-jointly standard deduction instead of $32,200, overstating taxable income by $9,800. It also submitted $5,236, which is inconsistent with the tax generated by its own stated $72,525 taxable income."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied a post-TCJA-expiration baseline with personal exemptions instead of the operative 2026 law represented in the trace. The correct computation takes the $32,200 standard deduction from $94,925.30 of AGI and applies the continuing 10% and 12% schedule."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed TCJA sunset parameters, using a $16,100 standard deduction, personal exemptions, and a restored 15% bracket. The operative 2026 calculation instead uses the $32,200 standard deduction and the 10%/12% schedule, yielding $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model stated the correct $94,925 AGI but supplied $8,993 without identifying the deduction or bracket arithmetic. With the $32,200 standard deduction, taxable income is $62,725.30, all of which lies within the applicable 10% and 12% brackets and produces $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model used unspecified sunset-rule deductions and personal exemptions rather than the operative $32,200 married-filing-jointly standard deduction. The correct taxable income is $62,725.30, to which the 2026 continuing rate schedule applies."
-us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,900 standard deduction instead of $32,200 and an estimated $24,600 first-bracket ceiling. These substitutions overstated taxable income by $1,300 and produced $7,191 instead of $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model erased the tax liability despite $94,925.30 of AGI and no available nonrefundable credits. After the $32,200 standard deduction, $62,725.30 remains taxable and generates $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $2,460 answer captures only about 10% of the first married-filing-jointly bracket and omits the 12% tax on the remaining taxable income. Taxable income is $62,725.30, so both bracket layers must be included."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that deductions and credits eliminate all taxable income. The household has $94,925.30 of AGI, only a $32,200 standard deduction, and no nonrefundable credits, leaving $62,725.30 taxable."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied post-TCJA-sunset rules, including a smaller standard deduction, personal exemptions, and a 15% bracket. The operative 2026 rules use a $32,200 standard deduction and the continuing 10% and 12% brackets for this income."
-us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used reverted-law parameters: a $16,800 standard deduction, $10,800 of personal exemptions, and a 15% bracket. Those provisions do not govern this computation; the $32,200 standard deduction and applicable 10%/12% schedule yield $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no numeric output or explanation for the requested variable, so the required structured answer was missing."
-us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model recognized taxable income near $63,425 and positive tax between $5,500 and $6,500, then set the liability to zero even after stating that no nonrefundable credits applied. With no credit offset, the bracket tax remains positive and equals $7,031.04 under the exact parameters."
-us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $31,450 standard deduction instead of $32,200 and a $24,350 first-bracket threshold instead of the applicable 2026 threshold. The correct taxable income is $62,725.30 and the exact schedule produces $7,031.04."
-us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used a fictitious $9,300 standard deduction, deducted medical expenses separately, and treated a purported senior bonus as a tax credit after computing tax. The head is 61, and the actual computation simply uses the $32,200 joint standard deduction, with no senior adjustment or nonrefundable credit."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning computed approximately $7,019 using nearly correct deduction and bracket estimates, then submitted $3,325 without any supporting tax step. The final value contradicts its own derivation and omits $3,694.04 of the liability."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $29,900 standard deduction instead of $32,200 and then misapplied the brackets even to its own $65,025 taxable-income estimate. The applicable deduction and rates produce $62,725.30 of taxable income and $7,031.04 of tax."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used approximate values of $32,300 for the standard deduction and $24,000 for the 10% bracket ceiling. The exact $32,200 deduction and 2026 bracket threshold produce $7,031.04 rather than $7,035."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly derived approximately $7,031 from the correct AGI, standard deduction, and bracket structure, then replaced it with $6,934 as an unsupported rounding adjustment. Rounding cannot remove $97.04 from the computed liability."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction instead of the applicable $32,200 deduction, leaving taxable income $2,200 too high. Its use of older approximate bracket thresholds compounded the overstatement, producing $7,314 instead of $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived $7,031 from $62,725 of taxable income and the relevant 10% and 12% brackets, then submitted $4,700 without a deduction or credit supporting that reduction. No nonrefundable credit applies to reduce the computed tax."
+us,scenario_025,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the $30,000 2025 standard deduction instead of the $32,200 deduction applicable in 2026. That left taxable income at $64,925 rather than $62,725.30 and overstated the tax."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the listed employer-sponsored insurance premium from AGI, invented mortgage interest and state taxes from balances or unstated facts, and applied personal exemptions instead of the $32,200 standard deduction. It then used a restored 15% bracket rather than the applicable 2026 12% bracket."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a $22,400 married-filing-jointly standard deduction instead of $32,200, so its stated taxable income was $72,525 rather than $62,725.30. Its $5,236 answer also does not follow from the applicable 2026 brackets on either taxable-income figure."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-expiration baseline with restored personal exemptions and rates. The applicable 2026 computation instead uses the $32,200 standard deduction and the 10% and 12% married-filing-jointly brackets, yielding $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied sunset-law parameters: a $16,100 standard deduction, personal exemptions, and a 15% second bracket. The applicable law uses a $32,200 standard deduction with no personal exemptions in this calculation and taxes the remaining income through the 12% bracket."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $8,993 implies use of expired-law or otherwise incorrect deduction and rate parameters. The stated $94,925 AGI must be reduced by the $32,200 standard deduction and taxed under the applicable 2026 brackets, producing $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model expressly used sunset rules with personal exemptions instead of the applicable 2026 standard-deduction and rate regime. The correct path uses taxable income of $62,725.30 and yields $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction at $30,900 rather than using $32,200, leaving taxable income $1,300 too high. Its approximate $24,600 bracket threshold also differed from the applicable 2026 threshold, resulting in $7,191."
+us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model erased all liability through unspecified deductions or nonrefundable credits even though it identified wages and pension as taxable income. The $32,200 standard deduction leaves $62,725.30 taxable, and no applicable nonrefundable credit reduces the resulting $7,031.04 to zero."
+us,scenario_025,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $2,460 answer captures only roughly the 10% tax on the first bracket and omits the 12% tax on taxable income above that threshold. After the $32,200 standard deduction, the remaining $62,725.30 spans both brackets."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared that deductions and credits eliminate $94,925 of wages and taxable pension. The standard deduction removes only $32,200, no nonrefundable credits apply, and $62,725.30 remains taxable."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used post-TCJA-sunset parameters, including a reduced standard deduction, personal exemptions, and a 15% second bracket. The applicable 2026 rules instead provide a $32,200 standard deduction and tax the remaining income at 10% and 12%."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied a $16,600 sunset standard deduction, two personal exemptions, and a 15% second bracket. Those are not the applicable 2026 parameters; the $32,200 standard deduction and 12% second bracket yield $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model imposed reverted-law parameters—a $16,800 standard deduction, personal exemptions, and a 15% bracket. The applicable computation uses the $32,200 standard deduction, no personal exemptions, and the 10% and 12% brackets."
+us,scenario_025,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_025,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model estimated a positive pre-credit tax of roughly $5,500–$6,500 and correctly identified that no nonrefundable credits apply, then nevertheless submitted zero. With no credit reduction, the bracket calculation remains positive and equals $7,031.04 under the applicable parameters."
+us,scenario_025,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $31,450 standard deduction instead of $32,200 and approximate bracket thresholds. This overstated taxable income by $750 and produced $7,130 rather than $7,031.04."
+us,scenario_025,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used an invented $9,300 standard deduction, separately deducted medical expenses, and treated a senior bonus deduction as a tax credit. The head is only 61, and the applicable computation simply subtracts the $32,200 joint standard deduction from AGI before applying the 2026 brackets."
 us,scenario_025,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_025,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model correctly stated that the 61-year-old head does not meet Medicare eligibility, but then submitted value = 1, which denotes Yes. It inverted its own eligibility conclusion when encoding the numeric output."
 us,scenario_025,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly added an Ohio employee payroll tax, even though Ohio has no mandatory employee-side state payroll tax included here, and invented an unexplained adjustment. It then submitted $9,432.76 despite its own displayed components totaling roughly $5,432.76; the federal Social Security and Medicare components alone total $4,798.48."
@@ -1375,136 +1443,148 @@ us,scenario_025,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,Fa
 us,scenario_025,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model stated the correct rule—7.65% employee payroll tax on the spouse’s $62,725 wages with no Additional Medicare Tax—but submitted $9,571.50 instead of performing that multiplication. The stated computation yields $4,798.46 before trace-level rounding, not the submitted amount."
 us,scenario_025,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model said it computed payroll tax on annual wages for both spouses even though only the spouse has listed wages and every unlisted numeric input must be zero. It therefore introduced a nonexistent wage base for the head instead of applying Social Security and Medicare tax solely to the spouse’s $62,725."
 us,scenario_025,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract rather than completing the calculation."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly identified the $200 retirement credit but replaced the 2026 Ohio schedule with a flat 2.75% calculation above $26,050 and then made an unsupported adjustment down to $875. The applicable schedule produces $2,121.57 before the credit and $1,921.57 after it."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly started Ohio taxable income from federal taxable income after a federal standard deduction; Ohio starts here from $94,925.30 of Ohio AGI and subtracts $3,800 of Ohio personal exemptions. It also used obsolete rates reaching 5.75% and omitted the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used the wrong $1,850 personal-exemption amount and explicitly applied no nonrefundable credit. Two $1,900 exemptions yield $91,125.30 of taxable income, and the qualifying pension generates a $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model reached the correct exemption deduction and recognized the retirement credit, but its flat 2.75%-above-$26,050 formula is not the applicable 2026 Ohio tax computation. The schedule produces $2,121.57 before the $200 credit, not the internally inconsistent $2,426 it submitted."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model deducted approximately $5,000 of personal exemptions instead of the correct $3,800 and used an incorrect simplified bracket formula. It also failed to separately apply the $200 retirement-income credit to the correctly computed $2,121.57 pre-credit tax."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used two $2,400 exemptions rather than two $1,900 exemptions, reducing taxable income by an extra $1,000. It then applied an obsolete flat 2.75% excess-income formula; the correct schedule gives $2,121.57 before the correctly identified $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model invented roughly $500 of retirement and senior credits, although the applicable nonrefundable reduction is only the $200 retirement-income credit. It also used $2,350 exemptions instead of $1,900 and an incorrect simplified tax schedule."
-us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted only one $2,400 exemption instead of two $1,900 exemptions and used the wrong graduated rates. It then omitted the $200 retirement-income credit despite $32,200 of qualifying taxable pension income."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the head's $21,208 employer-sponsored insurance premium from wages even though the stated $62,725 wage amount is the income input and no separate pretax wage deduction is authorized. It also invented an Ohio medical-expense deduction and used $4,800 rather than $3,800 of exemptions, severely understating taxable income."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $1,695 estimate does not follow the traced computation: $91,125.30 of taxable income produces $2,121.57 under the applicable Ohio schedule, followed by a $200 retirement credit. Its number reflects an approximate rate shortcut rather than the required bracket-and-credit calculation."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a flat 2.75% rate only to income above an approximate $26,050 threshold. That is not the applicable 2026 Ohio schedule, and it also failed to apply the $200 retirement-income credit after calculating pre-credit tax."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used invented $2,700-per-person exemptions instead of $1,900 per person and then applied the wrong flat 2.75% formula. It also omitted the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly obtained $91,125 of taxable income and recognized the $200 retirement credit, but it computed pre-credit tax as a flat 2.75% of income above $26,050. The applicable schedule produces $2,121.57 before that credit, yielding $1,921.57."
-us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's bare estimate does not implement the traced Ohio schedule and retirement-credit calculation. Tax on $91,125.30 is $2,121.57 before the $200 credit, not $1,694.06 after unspecified rates and credits."
-us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the federal standard deduction and $4,800 of Ohio exemptions from Ohio AGI. Ohio taxable income is $91,125.30 after only $3,800 of personal exemptions, and the model also omitted the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a low-income or family offset that does not apply to this childless couple with $94,925.30 of Ohio AGI. The Ohio schedule produces $2,121.57 before the sole applicable $200 retirement credit, leaving positive tax of $1,921.57."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked an unspecified Ohio standard deduction even though the traced calculation subtracts only two $1,900 personal exemptions from Ohio AGI. Its $1,340 estimate also fails to implement the applicable schedule and $200 retirement credit."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $230.625 of medical expenses from Ohio income. With no such deduction, taxable income is $91,125.30; applying the schedule and the correctly identified $200 retirement credit yields $1,921.57."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used two $2,400 exemptions rather than two $1,900 exemptions and applied an incorrect flat-rate formula. It also stated that no nonrefundable credits applied, omitting the $200 retirement-income credit generated by the $32,200 pension."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly used two $1,900 exemptions and the $200 retirement credit but calculated pre-credit tax as $2,150.25 instead of $2,121.57. It therefore misapplied the 2026 Ohio schedule to $91,125.30 of taxable income."
-us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the entire $32,200 private pension from Ohio taxable income even though it is included in the $94,925.30 Ohio AGI. Ohio instead taxes that pension and grants a $200 retirement-income credit; the model also used incorrect exemptions and invented joint-filing and $25 retirement credits."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model asserted that deductions and credits reduce Ohio taxable income or liability to zero without identifying any applicable provision. The household has $91,125.30 of taxable income and only a $200 retirement credit against $2,121.57 of pre-credit tax."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used $4,800 of personal exemptions instead of the correct $3,800 and applied the wrong flat 2.75% excess-income formula. It also explicitly omitted the qualifying $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used invented $2,650 exemptions and an inapplicable 3.5% bracket above $46,100. It also omitted the $200 retirement-income credit; the correct taxable income, scheduled tax, and credit are $91,125.30, $2,121.57, and $200."
-us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model deducted $4,800 of exemptions instead of $3,800 and used an incorrect flat 2.75% formula above $26,050. It also said no nonrefundable credit applied, overlooking the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used estimated $2,500 personal exemptions instead of the applicable $1,900 exemptions and calculated tax with an incorrect flat 2.75% formula. It also omitted the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly calculated $91,125 of taxable income and applied the $200 retirement credit, but its flat 2.75%-above-$26,050 computation understated pre-credit tax. The applicable schedule produces $2,121.57 before the credit and $1,921.57 after it."
-us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model invented senior or aged adjustments that reduce this couple's taxable income nearly to zero. The head's age does not erase $91,125.30 of Ohio taxable income, and the applicable reduction is the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied an unspecified estimated bracket calculation directly to $94,925 rather than subtracting $3,800 of personal exemptions and using the applicable schedule on $91,125.30. It also failed to account for the $200 retirement-income credit."
-us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $9,300 Ohio married-filing-jointly standard deduction, taxed the zero-rate band at 2.75%, and then applied an inapplicable $25 senior credit to a 61-year-old. The correct deductions are two $1,900 personal exemptions, and the applicable nonrefundable credit is the $200 retirement-income credit."
-us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model applied only North Carolina's CHIP income ceiling and concluded that income below 216% of FPL established eligibility. It omitted the prior Medicaid screen: Child 1 qualifies under the OLDER_CHILD Medicaid category, which makes the child ineligible for CHIP."
-us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model treated income below 211% of FPL as sufficient for CHIP eligibility and also used an incorrect household-income figure. It failed to apply the controlling Medicaid exclusion: Child 1 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP.
-us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_chip_eligible. The required output is 0 because Child 1 qualifies for Medicaid under the OLDER_CHILD category, and Medicaid eligibility excludes CHIP eligibility."
-us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 138% FPL limit instead of North Carolina's applicable 2026 OLDER_CHILD limit. At the traced MAGI of 2.13 times FPL, the 11-year-old satisfies that category's income test."
-us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,The model used an incorrect 200% FPL ceiling and an erroneous family-of-five poverty amount rather than the applicable OLDER_CHILD threshold. It also treated employer-sponsored insurance as evidence against Medicaid eligibility even though it does not disqualify the child under this rule.
-us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model substituted a 138% FPL ceiling for North Carolina's applicable 2026 OLDER_CHILD income limit. The traced 2.13-times-FPL MAGI passes the correct test.
-us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model applied an incorrect 138% FPL limit to the 11-year-old. North Carolina's OLDER_CHILD category permits eligibility at the traced MAGI of 2.13 times FPL.
-us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the relevant income level as about 260% FPL instead of using the engine's MAGI result of 2.13 times FPL. At 2.13 times FPL, the child is within the applicable OLDER_CHILD limit."
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented an uninsured requirement for Medicaid and treated employer-sponsored insurance as disqualifying. Existing employer-sponsored coverage does not prevent the child from satisfying the OLDER_CHILD Medicaid category.
-us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,The model added the separately listed FLSA overtime premium to gross wages even though the prompt states that gross wages already include overtime. That double count inflated income; the traced MAGI is 2.13 times FPL and falls within the OLDER_CHILD limit.
-us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model applied a 133% FPL threshold rather than North Carolina's applicable OLDER_CHILD limit. The traced MAGI of 2.13 times FPL qualifies under the correct category.
-us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model imposed an incorrect 147% FPL ceiling on an 11-year-old in the OLDER_CHILD category. The applicable North Carolina limit admits the traced 2.13-times-FPL MAGI.
-us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer implies use of an income ceiling below the applicable North Carolina OLDER_CHILD limit. The engine's 2.13-times-FPL MAGI satisfies that limit.
-us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model used a 133% FPL Medicaid limit and diverted the child to CHIP. The applicable OLDER_CHILD Medicaid threshold includes a child at 2.13 times FPL.
-us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model applied a 133% FPL threshold instead of the applicable North Carolina OLDER_CHILD limit. A MAGI of 2.13 times FPL passes the correct category-specific test.
-us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The answer implies an income limit below the applicable OLDER_CHILD threshold. North Carolina's category for ages 6 through 18 admits the traced MAGI of 2.13 times FPL.
-us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly concluded that household income exceeded the child Medicaid limit. Under the applicable OLDER_CHILD threshold, the traced 2.13-times-FPL MAGI qualifies."
-us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,The model treated employer-sponsored insurance as a Medicaid disqualifier. The OLDER_CHILD eligibility rule remains satisfied despite existing employer-sponsored coverage.
-us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model used an unspecified benchmark threshold below North Carolina's applicable OLDER_CHILD limit. The traced MAGI of 2.13 times FPL is within the correct limit.
-us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,The model inferred ineligibility from wages without applying the category-specific OLDER_CHILD income limit. The traced MAGI is 2.13 times FPL and satisfies that limit; no special status is required beyond the child's age and dependent status.
-us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model treated the Medicaid and CHIP income limits as a single ceiling below the household's income. The applicable OLDER_CHILD Medicaid limit admits the traced MAGI of 2.13 times FPL.
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated a North Carolina child Medicaid threshold that was too low. The applicable OLDER_CHILD threshold includes the child's traced 2.13-times-FPL MAGI.
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model incorrectly placed the household MAGI above North Carolina's limit for an 11-year-old. The applicable OLDER_CHILD limit admits MAGI at 2.13 times FPL.
-us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a child Medicaid limit below the applicable OLDER_CHILD threshold. At the traced 2.13-times-FPL MAGI, Child 1 meets the income test."
-us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly treated income as exceeding the applicable OLDER_CHILD limit and also counted employer-sponsored insurance against eligibility. The traced MAGI of 2.13 times FPL qualifies, and existing employer coverage does not disqualify the child."
-us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model applied a 133% FPL ceiling to the ages 6–18 category. North Carolina's applicable OLDER_CHILD limit permits eligibility at 2.13 times FPL.
-us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an approximately 133% FPL base limit instead of the applicable North Carolina OLDER_CHILD threshold. It also overstated the relevant income level; the trace establishes MAGI at 2.13 times FPL, which qualifies."
-us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,The model imposed a 133% FPL limit on the OLDER_CHILD category. The applicable limit admits the traced 2.13-times-FPL MAGI; CHIP's uninsured condition does not alter this Medicaid determination.
-us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, so the required output was missing."
-us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,The model applied a 138% FPL ceiling instead of the applicable OLDER_CHILD limit. It also overstated MAGI as about 226% FPL; the traced value is 2.13 times FPL and qualifies.
-us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model concluded that income exceeded the Medicaid-for-children limit without applying North Carolina's applicable OLDER_CHILD threshold. The traced MAGI of 2.13 times FPL is within that limit.
-us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model's own income comparison placed the household within its stated child Medicaid limit, but it then treated employer-sponsored insurance as disqualifying. Employer-sponsored coverage does not negate eligibility under the OLDER_CHILD Medicaid rule."
-us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model used an income ceiling below North Carolina's applicable OLDER_CHILD limit. The engine's MAGI result of 2.13 times FPL satisfies the correct category-specific threshold.
-us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated household MAGI below the CHIP income limit as sufficient for CHIP eligibility and omitted the prerequisite that the child not qualify for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes the CHIP result 0."
-us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model applied the NC Health Choice income threshold without first testing Medicaid eligibility. Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore excluded from CHIP, irrespective of its separate $61,093 income calculation."
-us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable output for child2_chip_eligible. The required derivation is Medicaid eligibility under the OLDER_CHILD category followed by CHIP exclusion, yielding 0."
-us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model used an older-child limit of about 211% FPL and independently estimated MAGI at 219% FPL. PolicyEngine's 2026 OLDER_CHILD parameters place the traced 2.13-times-FPL MAGI within the applicable North Carolina limit.
-us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,The model incorrectly treated available employer-sponsored insurance as an additional Medicaid disqualifier and also invented wages for the head despite unlisted income being zero. Child 2 qualifies through the OLDER_CHILD pathway at the traced MAGI of 2.13 times FPL regardless of existing employer coverage.
-us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model cycled among 215%, 211%, and 133% FPL limits and ultimately applied the wrong threshold to an 11-year-old. The applicable 2026 North Carolina OLDER_CHILD threshold includes the traced 2.13-times-FPL MAGI."
-us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model imposed a 138% Medicaid cutoff and treated the higher child limit as CHIP-only. PolicyEngine classifies this 11-year-old in the OLDER_CHILD Medicaid category, whose 2026 North Carolina limit covers MAGI at 2.13 times FPL."
-us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the household's income level as roughly 260% FPL. The Medicaid MAGI computation yields 2.13 times FPL, and that level satisfies North Carolina's OLDER_CHILD limit."
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented an uninsured-status requirement for Medicaid and treated employer-sponsored coverage as disqualifying. Existing employer coverage does not block Child 2's OLDER_CHILD Medicaid eligibility at 2.13 times FPL.
-us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,The model added the separately listed FLSA overtime premium to gross wages even though gross wages already include all overtime under the prompt. That double counting pushed income above the limit; the traced Medicaid MAGI is 2.13 times FPL and satisfies the OLDER_CHILD threshold.
-us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model applied the 133% FPL standard instead of North Carolina's applicable OLDER_CHILD limit. Child 2's traced MAGI of 2.13 times FPL remains within the correct 2026 category threshold.
-us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a 133% FPL cutoff to the wrong child eligibility standard. Although it derived a lower income after deductions, it never compared the resulting MAGI with the applicable North Carolina OLDER_CHILD threshold, which covers 2.13 times FPL."
-us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The model asserted that income exceeded the Medicaid limit without applying the 2026 North Carolina OLDER_CHILD threshold. The traced MAGI is 2.13 times FPL and falls within that limit.
-us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model used a 133% FPL Medicaid cutoff and diverted higher-income children to CHIP. Child 2 instead qualifies under PolicyEngine's OLDER_CHILD Medicaid category because 2.13 times FPL is within its applicable limit.
-us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model compared its $61,093 income estimate with an inapplicable 133% FPL threshold. The governing OLDER_CHILD threshold is higher and admits the traced MAGI of 2.13 times FPL."
-us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The model declared the household over the income limit without applying the OLDER_CHILD threshold. North Carolina's applicable 2026 limit covers Child 2 at the traced 2.13-times-FPL MAGI.
-us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted excess income without identifying or applying the category-specific limit. Child 2 is an OLDER_CHILD, and 2.13 times FPL is within the applicable North Carolina threshold."
-us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated 133% FPL as the Medicaid ceiling for ages 6–18 and 210% as a separate CHIP ceiling. PolicyEngine's 2026 OLDER_CHILD Medicaid threshold covers the traced 2.13-times-FPL MAGI, and employer-sponsored insurance does not negate eligibility."
-us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model relied on an unspecified benchmark Medicaid threshold instead of the North Carolina OLDER_CHILD limit. Applying the correct category threshold to MAGI of 2.13 times FPL yields eligibility.
-us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an unspecified special status and failed to recognize that being age 11 and a tax-unit dependent places Child 2 in the OLDER_CHILD MAGI category. That pathway's income test is satisfied at 2.13 times FPL.
-us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model concluded that earnings exceeded the child Medicaid ceiling without applying the 2026 OLDER_CHILD parameter. The traced MAGI is 2.13 times FPL, which is within the applicable North Carolina limit."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an estimated child Medicaid threshold that was lower than the applicable OLDER_CHILD limit. At the engine-calculated 2.13 times FPL, Child 2 passes North Carolina's 2026 income test."
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model compared household MAGI with the wrong North Carolina Medicaid income limit for an 11-year-old. The correct OLDER_CHILD threshold includes MAGI at 2.13 times FPL.
-us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model asserted that household income exceeded the child limit without using the OLDER_CHILD category parameter. The traced 2.13-times-FPL MAGI is within North Carolina's applicable 2026 limit.
-us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model applied an income ceiling below the applicable OLDER_CHILD threshold and also treated employer-sponsored insurance as adverse to eligibility. Child 2 passes the correct income test at 2.13 times FPL, and ESI does not disqualify the child from Medicaid."
-us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model used 133% FPL as the Medicaid limit for Child 2. The applicable 2026 North Carolina OLDER_CHILD threshold is higher and covers the traced MAGI of 2.13 times FPL.
-us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated household income as roughly 240% FPL and then applied a 133% base cutoff. The Medicaid computation produces MAGI of 2.13 times FPL, which is within the OLDER_CHILD limit."
-us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model imposed a 133% FPL Medicaid limit on an 11-year-old and compounded the error by invoking employer-sponsored insurance. North Carolina's applicable OLDER_CHILD Medicaid threshold covers 2.13 times FPL, and ESI does not defeat eligibility."
-us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for child2_medicaid_eligible. It therefore failed the required output contract rather than completing the OLDER_CHILD eligibility calculation.
-us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,The model applied a 133%/138% FPL ceiling rather than the applicable North Carolina OLDER_CHILD threshold. Child 2's traced MAGI of 2.13 times FPL passes the correct category-specific test.
-us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model asserted that income was well above the Medicaid-for-children limit without applying the 2026 OLDER_CHILD threshold. The traced income level is 2.13 times FPL and satisfies that threshold.
-us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 133% FPL ceiling for ages 6–18 and an incorrect five-person FPL estimate. The governing OLDER_CHILD threshold covers the engine-calculated MAGI of 2.13 times FPL, while employer-sponsored insurance does not disqualify Child 2."
-us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared income too high without performing the category-specific comparison. Under North Carolina's OLDER_CHILD pathway, the traced MAGI of 2.13 times FPL is within the 2026 limit."
-us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model treated an asserted CHIP income band as sufficient for eligibility and never applied the rule excluding children who qualify for Medicaid. Child 3 is Medicaid-eligible under the OLDER_CHILD category and therefore is not CHIP-eligible.
-us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model concluded that income below its asserted CHIP limit established eligibility, omitting the prerequisite that the child not qualify for Medicaid. Child 3's OLDER_CHILD Medicaid eligibility automatically precludes CHIP."
-us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model used household income below the CHIP threshold as the complete eligibility test. It failed to apply the Medicaid-exclusion step: Child 3 qualifies for Medicaid under the OLDER_CHILD category and thus cannot qualify for CHIP.
-us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model checked age, Medicare status, and a CHIP income range but omitted the controlling Medicaid-status test. Child 3 is Medicaid-eligible under the OLDER_CHILD category, which makes the child ineligible for CHIP even when the asserted CHIP income test is satisfied."
-us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model applied a 138% FPL Medicaid ceiling and assigned income above it to CHIP. North Carolina's applicable OLDER_CHILD Medicaid limit covers child3 at 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,The model treated 207% FPL as the Medicaid ceiling and therefore rejected the child based on its own dollar conversion. The applicable OLDER_CHILD threshold covers the engine-calculated MAGI of 2.13 times FPL; employer-sponsored insurance does not alter this eligibility result.
-us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model used an older-child income limit of roughly 211% FPL and compared it with an inaccurate gross-wage shortcut. The applicable North Carolina OLDER_CHILD rule admits the engine-calculated MAGI of 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model substituted the historical 133% plus 5% disregard limit for North Carolina's applicable 2026 OLDER_CHILD threshold. Child3 qualifies in that category at the traced MAGI of 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the household's income level as about 260% FPL instead of using the traced MAGI of 2.13 times FPL. At the correct MAGI, child3 satisfies North Carolina's OLDER_CHILD income test."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly treated employer-sponsored insurance as a Medicaid disqualifier and used a 210% FPL ceiling. This output tests eligibility rather than enrollment or other coverage, and the OLDER_CHILD category covers the traced 2.13-times-FPL MAGI."
-us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the listed FLSA overtime premium to gross wages even though gross wages are annual totals including overtime, thereby double-counting income. It also imposed a 138% FPL older-child ceiling instead of applying the North Carolina OLDER_CHILD threshold that covers 2.13 times FPL."
-us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model applied a 133% FPL ceiling to a nine-year-old's Medicaid category. North Carolina's applicable OLDER_CHILD threshold covers child3 at the traced MAGI of 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model rejected child3 using a 133% FPL threshold. The applicable North Carolina OLDER_CHILD limit covers the traced MAGI of 2.13 times FPL, regardless of the model's unsupported $61,093 income figure."
-us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer implies an income ceiling below the household's traced 2.13-times-FPL MAGI. North Carolina's OLDER_CHILD category permits eligibility at that income level.
-us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model applied a 133% FPL older-child Medicaid limit and diverted the child to CHIP. The applicable OLDER_CHILD Medicaid threshold covers child3 at 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used 133% FPL as the controlling income ceiling. North Carolina's applicable OLDER_CHILD limit covers the engine-calculated 2.13-times-FPL MAGI; its unexplained $61,093 income figure does not change that rule."
-us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The model's conclusion assumes an older-child Medicaid limit below 2.13 times FPL. The applicable North Carolina OLDER_CHILD threshold includes child3 at that MAGI level.
-us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model treated the household's income as exceeding the controlling Medicaid limit. Under North Carolina's OLDER_CHILD category, the traced MAGI of 2.13 times FPL is within the limit."
-us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model used a 133% FPL Medicaid ceiling and a separate 210% FPL CHIP ceiling instead of the applicable OLDER_CHILD Medicaid parameter. Child3 qualifies at the traced 2.13-times-FPL MAGI, and employer-sponsored insurance does not negate eligibility."
-us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model invoked an unspecified benchmark threshold below the household's income. The controlling North Carolina OLDER_CHILD threshold covers child3 at the traced MAGI of 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an unspecified special-status fact instead of applying the ordinary age-based OLDER_CHILD pathway. Child3 is age 9 and qualifies through that category at 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model concluded that earnings exceeded North Carolina's child Medicaid limit without applying the applicable OLDER_CHILD threshold. The traced MAGI is 2.13 times FPL and falls within that category's limit.
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model used an estimated child Medicaid threshold below the traced income level. North Carolina's OLDER_CHILD limit covers child3 at 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model treated child3's MAGI as exceeding the age-nine Medicaid ceiling. The applicable North Carolina OLDER_CHILD threshold includes the traced 2.13-times-FPL MAGI.
-us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model assumed a North Carolina child Medicaid limit below the household's traced MAGI. The applicable OLDER_CHILD threshold covers child3 at 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model used an income ceiling below 2.13 times FPL and also treated employer-sponsored insurance as adverse to eligibility. The OLDER_CHILD category covers child3 at that MAGI, and existing employer coverage does not disqualify the child from this eligibility output."
-us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model substituted a 133% FPL ceiling for the applicable North Carolina OLDER_CHILD threshold. Child3 qualifies at the traced MAGI of 2.13 times FPL.
-us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated household income as roughly 240% FPL and applied a 133% FPL ceiling. The trace produces MAGI of 2.13 times FPL, which satisfies North Carolina's OLDER_CHILD income limit."
-us,scenario_026,child3_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit and cited employer-sponsored coverage as an additional bar. The applicable OLDER_CHILD threshold covers 2.13 times FPL, and employer-sponsored insurance does not negate Medicaid eligibility."
-us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for child3_medicaid_eligible. It therefore failed the required output contract rather than completing the eligibility determination.
-us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,The model used a 133% or 138% FPL ceiling for ages 6–18 and overstated MAGI as roughly 226% FPL. The traced MAGI is 2.13 times FPL and falls within North Carolina's applicable OLDER_CHILD limit.
-us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model asserted that income exceeded the Medicaid for Children limit without applying the controlling OLDER_CHILD parameter. The traced 2.13-times-FPL MAGI is within that limit.
-us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,The model used a 133% FPL ceiling and an incorrect poverty-guideline conversion for a five-person household. North Carolina's applicable OLDER_CHILD threshold covers child3 at the traced MAGI of 2.13 times FPL; employer-sponsored insurance does not reverse eligibility.
-us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model assumed that the household's income exceeded the relevant child Medicaid ceiling. The applicable North Carolina OLDER_CHILD limit includes child3 at the traced MAGI of 2.13 times FPL.
+us,scenario_025,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used an incorrect personal-exemption amount and then replaced the statutory 2026 schedule with 2.75% of income above $26,050. Its final unexplained reduction to $875 has no applicable credit behind it; the schedule yields $2,121.57 before the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model incorrectly started Ohio taxable income from federal taxable income after a federal standard deduction and applied obsolete rates reaching 5.75%. Ohio starts from $94,925.30 of AGI, subtracts $3,800 of Ohio exemptions, and applies the 2026 Ohio schedule before the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used the wrong $1,850 exemption amount, estimated rather than applied the 2026 schedule, and explicitly omitted the applicable $200 retirement-income credit. The correct sequence is $91,125.30 of taxable income, $2,121.57 of tax, then the $200 credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly identified $3,800 of exemptions and the $200 retirement credit but computed the bracket tax as only 2.75% of taxable income above $26,050. That shortcut omits the schedule's base-tax component and contradicts its own stated arithmetic, which cannot produce $2,426."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used approximately $5,000 of exemptions instead of $3,800 and did not apply the 2026 Ohio schedule or the $200 retirement credit explicitly. The correct taxable income is $91,125.30 and its scheduled tax is $2,121.57 before that credit."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used $2,400 per personal exemption instead of the applicable $1,900 and treated the tax as only 2.75% of income above $26,050. Although it correctly applied the $200 retirement credit, the correct pre-credit scheduled tax is $2,121.57."
+us,scenario_025,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model used incorrect exemptions, omitted the schedule's base-tax component, and invented roughly $500 of senior and retirement credits. Neither spouse qualifies for an age-based senior credit, while the applicable retirement-income credit is exactly $200."
+us,scenario_025,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model subtracted only one $2,400 exemption, used obsolete graduated rates of 2.75%, 3.226%, and 3.688%, and omitted the $200 retirement credit. The applicable computation uses two $1,900 exemptions and the 2026 schedule, producing $2,121.57 before the credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $21,208 of employer-sponsored insurance premiums from wages already stated as gross wages and then created an Ohio medical-expense deduction. Those reductions do not enter this Ohio AGI computation; AGI remains $94,925.30 before $3,800 of personal exemptions."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $1,695 estimate does not apply the traced Ohio computation. Tax on $91,125.30 under the 2026 schedule is $2,121.57, and the sole applicable nonrefundable retirement credit reduces it by $200."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used the shortcut 2.75% of income above the zero-tax threshold and stopped before applying the retirement credit. That shortcut omits the schedule's base-tax component; the scheduled tax is $2,121.57 before the $200 credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted $5,400 of personal exemptions instead of $3,800, reduced the tax schedule to 2.75% of excess income, and omitted the $200 retirement credit. Ohio taxable income is $91,125.30, not $89,525."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly found $91,125 of taxable income and the $200 retirement credit but treated the tax as only 2.75% of income above $26,050. The 2026 schedule includes a base-tax component and produces $2,121.57 before the credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's unsupported $1,694.06 does not implement the 2026 Ohio schedule on $91,125.30 of taxable income. That schedule yields $2,121.57 before subtracting the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the federal standard deduction in calculating Ohio taxable income and also used $4,800 rather than $3,800 of Ohio exemptions. Ohio begins with federal AGI, not federal taxable income, leaving $91,125.30 subject to the state schedule."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invented a complete offset based on a nonexistent low-income or family credit. The household has $2,121.57 of scheduled Ohio tax, and the applicable $200 retirement credit leaves $1,921.57."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $1,340 estimate reflects neither the applicable $3,800 exemption deduction nor the traced 2026 tax schedule and $200 retirement credit. Those steps produce $91,125.30 of taxable income and $1,921.57 after the credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model invented a $230.625 medical deduction, reducing taxable income below the correct $91,125.30. Medical expenses do not reduce Ohio AGI in this computation; applying the schedule to the undiminished base and then subtracting $200 yields $1,921.57."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model used $2,400 exemptions, reduced the schedule to 2.75% above $26,050, and explicitly omitted the applicable $200 retirement credit. The correct exemptions are $1,900 each and scheduled tax is $2,121.57 before that credit."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly used two $1,900 exemptions and the $200 retirement credit but calculated the pre-credit schedule as $2,150.25 instead of $2,121.57. Applying the 2026 schedule to $91,125.30 fixes the $28.68 overstatement."
+us,scenario_025,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the entire $32,200 private pension from Ohio taxable income even though it is taxable pension income. Ohio taxes it in AGI and provides a $200 nonrefundable retirement credit instead of an income exclusion."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model asserted that unspecified deductions and credits eliminate all taxable income. Only $3,800 of personal exemptions and a $200 retirement credit apply in the traced calculation, leaving $1,921.57 of tax."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used $4,800 rather than $3,800 of exemptions, treated the schedule as 2.75% above $26,050, and omitted the $200 retirement credit. The correct taxable income is $91,125.30 and scheduled tax is $2,121.57 before the credit."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model correctly identified the exemptions and retirement credit but omitted the schedule's base-tax component and wrongly applied a 5% joint-filing credit when only the spouse has qualifying income. The tax is $2,121.57 before the valid $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used $5,300 of exemptions instead of $3,800, imposed an obsolete 3.5% bracket at this income, and omitted the $200 retirement credit. The applicable 2026 schedule on $91,125.30 yields $2,121.57 before that credit."
+us,scenario_025,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used $4,800 instead of $3,800 of personal exemptions, omitted the schedule's base-tax component, and stated that no nonrefundable credit applied. The $32,200 taxable pension qualifies the household for the $200 retirement-income credit."
+us,scenario_025,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used estimated exemptions of $2,500 each instead of $1,900 each, reduced the schedule to 2.75% above $26,050, and omitted the $200 retirement credit. The correct taxable income is $91,125.30."
+us,scenario_025,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly determined the exemption deduction, taxable income, and retirement credit but treated the schedule as only 2.75% of income above $26,050. That omits the schedule's base-tax component, so pre-credit tax is $2,121.57 rather than $1,789.56."
+us,scenario_025,state_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented senior or aged adjustments that reduce taxable income to zero. Neither spouse is 65, and the applicable deductions and credits are limited to $3,800 of personal exemptions and the $200 retirement credit."
+us,scenario_025,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the two $1,900 Ohio personal exemptions and omitted the $200 retirement-income credit. It also reduced the schedule to 2.75% above $26,050 instead of applying the full scheduled tax computation."
+us,scenario_025,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model supplied an unsupported bracket estimate of $2,651 without subtracting the traced $3,800 exemption deduction or $200 retirement credit. The 2026 schedule on $91,125.30 produces $2,121.57 before the credit."
+us,scenario_025,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a $9,300 Ohio MFJ standard deduction, applied 2.75% to the zero-rate portion and 3.5% above it, and granted a senior credit to a 61-year-old. Ohio instead subtracts $3,800 of personal exemptions, applies the 2026 schedule to $91,125.30, and grants the $200 retirement credit."
+us,scenario_026,child1_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model applied only the CHIP income ceiling and failed to determine Medicaid eligibility first. Child 1 qualifies for Medicaid under the OLDER_CHILD category, which makes the child ineligible for CHIP regardless of being below the asserted CHIP limit."
+us,scenario_026,child1_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated income below 211% FPL as sufficient for CHIP eligibility and omitted the requirement that the child not qualify for Medicaid. Child 1 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false."
+us,scenario_026,child1_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model incorrectly used 133% FPL as Child 1's Medicaid cutoff and therefore placed the child into a supposed gap between Medicaid and CHIP thresholds. The applicable OLDER_CHILD Medicaid pathway covers Child 1, and that Medicaid eligibility precludes CHIP."
+us,scenario_026,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_chip_eligible, violating the required output contract. The required result is 0 because Child 1 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP."
+us,scenario_026,child1_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model applied a 138% FPL limit to North Carolina's OLDER_CHILD category instead of the 2026 limit used for children ages 6–18. It therefore rejected the child even though the relevant MAGI is 2.13 times FPL and falls within that category's limit.
+us,scenario_026,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an erroneous 200% FPL cutoff and an incorrect poverty guideline for a five-person household, rather than evaluating the traced MAGI of 2.13 times FPL under North Carolina's OLDER_CHILD limit. It also incorrectly treated access to employer-sponsored insurance as evidence against Medicaid eligibility."
+us,scenario_026,child1_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model imposed a 138% FPL ceiling on the OLDER_CHILD category. North Carolina's applicable 2026 older-child limit permits eligibility at the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used a 138% FPL income limit for an 11-year-old instead of North Carolina's applicable 2026 OLDER_CHILD threshold. At the correctly derived 2.13-times-FPL MAGI, Child 1 passes the category's income test."
+us,scenario_026,child1_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the household's relevant MAGI as roughly 260% FPL. The traced MAGI is 2.13 times FPL, which is within North Carolina's OLDER_CHILD income limit."
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model invented an uninsured requirement for Medicaid and treated employer-sponsored insurance as disqualifying. Existing employer-sponsored coverage does not prevent Child 1 from qualifying through North Carolina's income-tested OLDER_CHILD pathway.
+us,scenario_026,child1_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the listed FLSA overtime premium to gross wages even though the prompt states that gross wages already include overtime pay, inflating household income toward $93,000. The engine derives MAGI of 2.13 times FPL, which passes the OLDER_CHILD limit."
+us,scenario_026,child1_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model applied a 133% FPL ceiling to North Carolina's older-child Medicaid category. The applicable 2026 OLDER_CHILD limit covers the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model used an approximately 147% FPL limit for the 11-year-old's category. North Carolina's applicable OLDER_CHILD threshold permits eligibility at the engine-derived MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an income ceiling below the traced MAGI of 2.13 times FPL. North Carolina's 2026 OLDER_CHILD income limit includes that MAGI level, so the income test is satisfied."
+us,scenario_026,child1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model incorrectly capped Medicaid for older children at 133% FPL and diverted the child to CHIP. The applicable North Carolina OLDER_CHILD Medicaid limit covers the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model imposed a 133% FPL threshold on Child 1's Medicaid category. The applicable 2026 OLDER_CHILD limit permits the child at the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer assumes that North Carolina's older-child income ceiling is below 2.13 times FPL. The applicable OLDER_CHILD limit includes 2.13 times FPL, so Child 1 satisfies the income test."
+us,scenario_026,child1_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model treated the household as above the older-child Medicaid income limit. The traced MAGI is 2.13 times FPL, which is within North Carolina's applicable OLDER_CHILD threshold."
+us,scenario_026,child1_medicaid_eligible,glm-5.2,llm_error,health_coverage,False,The model treated employer-sponsored insurance as a Medicaid disqualifier. Child 1 remains eligible through North Carolina's OLDER_CHILD pathway because existing private coverage does not negate the categorical and MAGI tests.
+us,scenario_026,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model applied an unspecified benchmark income threshold below the household's traced level. North Carolina's applicable OLDER_CHILD threshold covers MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an unstated special status and failed to apply the ordinary OLDER_CHILD pathway for an 11-year-old dependent. Under that pathway, the traced MAGI of 2.13 times FPL passes North Carolina's income limit."
+us,scenario_026,child1_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,The model conflated or misapplied the Medicaid and CHIP income limits for school-age children. North Carolina's OLDER_CHILD Medicaid limit permits Child 1 at the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated a North Carolina child Medicaid threshold below 2.13 times FPL. The applicable 2026 OLDER_CHILD limit includes the engine-derived 2.13-times-FPL MAGI.
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model incorrectly concluded that the household's MAGI exceeds the limit for an 11-year-old. The traced MAGI is 2.13 times FPL and falls within North Carolina's OLDER_CHILD threshold.
+us,scenario_026,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model placed the North Carolina child Medicaid limit below the household's relevant income. The applicable OLDER_CHILD limit covers the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly found the income above North Carolina's OLDER_CHILD limit and also treated employer-sponsored insurance as adverse to Medicaid eligibility. The traced MAGI is 2.13 times FPL, which passes the category's income test, and ESI does not disqualify the child."
+us,scenario_026,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model used a 133% FPL cutoff for North Carolina Medicaid for children ages 6–18. The applicable 2026 OLDER_CHILD threshold permits eligibility at 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,The model imposed a 133% FPL ceiling on the OLDER_CHILD category and used that ceiling to reject the child. North Carolina's applicable 2026 limit covers the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an approximately 133% FPL base limit for an 11-year-old and also overstated the relevant income as roughly 240% FPL. The engine derives MAGI of 2.13 times FPL, which is within the applicable OLDER_CHILD limit."
+us,scenario_026,child1_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model capped the Medicaid pathway for ages 6–18 at 133% FPL instead of applying North Carolina's applicable 2026 OLDER_CHILD limit. Its statement that CHIP requires being uninsured is irrelevant to Child 1's Medicaid eligibility, which is established at 2.13 times FPL."
+us,scenario_026,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for child1_medicaid_eligible. It therefore failed the required structured-output contract rather than completing the Medicaid eligibility computation.
+us,scenario_026,child1_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit, or 138% after a disregard, to North Carolina's OLDER_CHILD category. It also overstated the relevant MAGI as about 226% FPL; the traced value is 2.13 times FPL and passes the applicable limit."
+us,scenario_026,child1_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model assumed that the household was above North Carolina's Medicaid-for-children limit without applying the applicable OLDER_CHILD threshold. The traced MAGI of 2.13 times FPL is within that limit.
+us,scenario_026,child1_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,The model used a 133% FPL limit for North Carolina's age-6–18 Medicaid category. The applicable 2026 OLDER_CHILD limit permits eligibility at the traced MAGI of 2.13 times FPL.
+us,scenario_026,child1_medicaid_eligible,qwen-3.7-max,llm_error,health_coverage,False,"The model's own income comparison placed the household within its stated child Medicaid limit, but it then incorrectly disqualified Child 1 for having employer-sponsored insurance. Existing employer-sponsored coverage does not bar eligibility through North Carolina's OLDER_CHILD Medicaid pathway."
+us,scenario_026,child1_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The answer implies an older-child Medicaid limit below the household's traced MAGI. North Carolina's applicable OLDER_CHILD threshold includes 2.13 times FPL, so Child 1 passes the income test."
+us,scenario_026,child2_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated income below 216% FPL as sufficient for CHIP and omitted the prerequisite that the child not qualify for Medicaid. Child 2 qualifies for Medicaid under the OLDER_CHILD category, which makes the child ineligible for CHIP."
+us,scenario_026,child2_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model applied the NC Health Choice income threshold as a standalone eligibility test and never checked Medicaid eligibility first. Child 2's OLDER_CHILD Medicaid eligibility excludes the child from CHIP.
+us,scenario_026,child2_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility from placing MAGI between 133% and 211% FPL, but failed to apply Medicaid precedence. Child 2 is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP."
+us,scenario_026,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer for child2_chip_eligible. The required output is 0 because Child 2 qualifies for Medicaid under the OLDER_CHILD category, and that Medicaid eligibility excludes CHIP eligibility."
+us,scenario_026,child2_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,"The model used an approximately 211% FPL ceiling and calculated the household at about 219% FPL. The engine places MAGI at 2.13 times FPL and within North Carolina's OLDER_CHILD Medicaid limit, so that threshold comparison was wrong."
+us,scenario_026,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_coverage,False,The model treated employer-sponsored insurance as an additional reason to deny Medicaid and also invented wages for the head despite unlisted income being zero. Employer-sponsored coverage does not disqualify child2 from the income-qualified OLDER_CHILD Medicaid pathway.
+us,scenario_026,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model cycled among 215%, 211%, and 133% FPL limits and ultimately applied the 133% adult-style ceiling to an 11-year-old. Child2 belongs to the OLDER_CHILD category, whose applicable North Carolina threshold includes MAGI at 2.13 times FPL."
+us,scenario_026,child2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model split children's coverage into a 138% Medicaid limit and a higher CHIP limit, then compared an unreduced income figure with those ceilings. PolicyEngine applies North Carolina's OLDER_CHILD Medicaid threshold directly, and 2.13 times FPL is within it."
+us,scenario_026,child2_medicaid_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model overstated the household income level as roughly 260% FPL. The relevant MAGI is 2.13 times FPL, which satisfies the OLDER_CHILD Medicaid income test."
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,The model imposed an uninsured-status requirement on Medicaid and denied eligibility solely because child2 has employer-sponsored insurance. The OLDER_CHILD Medicaid pathway turns on categorical status and MAGI here; existing employer coverage does not bar eligibility.
+us,scenario_026,child2_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,The model added the separately reported FLSA overtime premium to annual wages even though gross wages already include overtime under the prompt. That double counting inflated income and led it to place the household above the child Medicaid limit instead of using the engine's 2.13-times-FPL MAGI.
+us,scenario_026,child2_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 133% FPL ceiling to an 11-year-old. Child2 is evaluated under North Carolina's OLDER_CHILD category, whose applicable threshold includes MAGI at 2.13 times FPL."
+us,scenario_026,child2_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 133% FPL Medicaid limit for the older-child category. North Carolina's applicable OLDER_CHILD threshold includes the traced MAGI of 2.13 times FPL, so its lower ceiling produced the wrong denial."
+us,scenario_026,child2_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies that the model compared household income with a child Medicaid ceiling below 2.13 times FPL. The applicable OLDER_CHILD limit includes 2.13 times FPL, so the income test is satisfied."
+us,scenario_026,child2_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model applied the 133% FPL limit to child2 and diverted higher-income coverage to CHIP. PolicyEngine's North Carolina OLDER_CHILD Medicaid threshold itself includes the household's 2.13-times-FPL MAGI.
+us,scenario_026,child2_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model denied child2 under a 133% FPL ceiling. That is not the applicable OLDER_CHILD threshold; the relevant North Carolina limit includes MAGI at 2.13 times FPL.
+us,scenario_026,child2_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer implies an income ceiling below the household's 2.13-times-FPL MAGI. North Carolina's OLDER_CHILD Medicaid threshold includes that income level, so child2 passes the income test."
+us,scenario_026,child2_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model treated the household as over the applicable Medicaid limit. Under the OLDER_CHILD category, the traced 2.13-times-FPL MAGI is within North Carolina's threshold."
+us,scenario_026,child2_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model applied a 133% FPL Medicaid ceiling and a separate 210% CHIP ceiling, then treated 213% FPL as exceeding both. The applicable OLDER_CHILD Medicaid threshold includes 2.13 times FPL, and employer-sponsored insurance does not undo that eligibility."
+us,scenario_026,child2_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model used an unspecified Medicaid benchmark below the household's income. The correct OLDER_CHILD comparison places MAGI at 2.13 times FPL within North Carolina's eligibility threshold.
+us,scenario_026,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an unspecified special status and failed to recognize that age 11 and dependent status place child2 in the OLDER_CHILD category. That category's income threshold includes the household's 2.13-times-FPL MAGI.
+us,scenario_026,child2_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model treated annual earnings after limited deductions as exceeding North Carolina's child Medicaid ceiling. The relevant comparison is the OLDER_CHILD threshold against MAGI of 2.13 times FPL, which passes."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model estimated a North Carolina child Medicaid threshold below 2.13 times FPL. The applicable OLDER_CHILD threshold includes that MAGI level, so its income denial was wrong."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model concluded that MAGI exceeded the limit for an 11-year-old. Under North Carolina's OLDER_CHILD category, MAGI at 2.13 times FPL is within the applicable limit."
+us,scenario_026,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model placed household income above the child Medicaid limit without applying the OLDER_CHILD threshold. That threshold includes the traced MAGI of 2.13 times FPL.
+us,scenario_026,child2_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model relied on both an understated income threshold and the presence of employer-sponsored insurance. Child2's 2.13-times-FPL MAGI meets the OLDER_CHILD limit, and employer coverage is not a Medicaid disqualifier."
+us,scenario_026,child2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model applied a 133% FPL ceiling to children ages 6–18. The governing North Carolina OLDER_CHILD threshold includes child2 at 2.13 times FPL.
+us,scenario_026,child2_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model compared its $74,555 MAGI estimate with a 133% FPL ceiling. The applicable OLDER_CHILD threshold is higher and includes the engine's 2.13-times-FPL MAGI."
+us,scenario_026,child2_medicaid_eligible,grok-build-0.1,llm_error,thresholds_rates,False,The model both overstated the household ratio as roughly 240% FPL and used a 133% base ceiling. The traced MAGI is 2.13 times FPL and falls within North Carolina's OLDER_CHILD Medicaid threshold.
+us,scenario_026,child2_medicaid_eligible,inkling,llm_error,thresholds_rates,False,"The model applied a 133% FPL limit to child2 and also treated employer-sponsored insurance as relevant to coverage eligibility. The OLDER_CHILD Medicaid threshold includes 2.13 times FPL, and existing employer coverage does not bar Medicaid eligibility."
+us,scenario_026,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child2_medicaid_eligible, so the required output was missing."
+us,scenario_026,child2_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used a 133% or 138% FPL ceiling and placed household MAGI at roughly 226% FPL. The engine's relevant MAGI is 2.13 times FPL, and North Carolina's OLDER_CHILD threshold includes that level."
+us,scenario_026,child2_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model treated household income as above North Carolina's Medicaid-for-children limit. The applicable OLDER_CHILD threshold includes MAGI at 2.13 times FPL, so the income test is met."
+us,scenario_026,child2_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,The model assigned a 133% FPL limit to the age-6-to-18 group. Child2's governing OLDER_CHILD threshold includes the relevant 2.13-times-FPL MAGI.
+us,scenario_026,child2_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 133% FPL ceiling for an 11-year-old and cited employer-sponsored insurance as supporting denial. North Carolina's OLDER_CHILD threshold includes MAGI at 2.13 times FPL, and employer coverage does not disqualify the child."
+us,scenario_026,child2_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model treated household income as exceeding the relevant child Medicaid limit. The OLDER_CHILD threshold includes the household's traced MAGI of 2.13 times FPL.
+us,scenario_026,child3_chip_eligible,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated the asserted 133%–211% FPL CHIP income band as sufficient and omitted the prerequisite that a child must not qualify for Medicaid. Child 3 qualifies for Medicaid under the OLDER_CHILD category, which disqualifies the child from CHIP."
+us,scenario_026,child3_chip_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model applied only its asserted 216% FPL CHIP ceiling and failed to test Medicaid eligibility first. Child 3's Medicaid eligibility under the OLDER_CHILD category bars CHIP eligibility even when income is below the cited CHIP limit.
+us,scenario_026,child3_chip_eligible,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model concluded that income below 211% FPL established CHIP eligibility, omitting CHIP's exclusion of children who already qualify for Medicaid. Child 3 is Medicaid-eligible under the OLDER_CHILD category and therefore is not CHIP-eligible."
+us,scenario_026,child3_chip_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model relied on age, lack of Medicare eligibility, and placement within a CHIP income range but never applied the controlling Medicaid-ineligibility condition. Child 3 qualifies for Medicaid under the OLDER_CHILD category, so CHIP eligibility is false."
+us,scenario_026,child3_chip_eligible,grok-4.6,llm_error,categorical_eligibility,False,The model treated being age 9 with MAGI between 133% and 211% FPL as sufficient for CHIP and skipped the Medicaid eligibility screen. The child's OLDER_CHILD Medicaid eligibility automatically excludes the child from CHIP.
+us,scenario_026,child3_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model explicitly asserted that Child 3 was not Medicaid-eligible, reversing the decisive eligibility result. Child 3 is Medicaid-eligible under the OLDER_CHILD category, and that status disqualifies the child from CHIP regardless of the model's household-size and CHIP-threshold calculation."
+us,scenario_026,child3_medicaid_eligible,claude-fable-5,llm_error,thresholds_rates,False,The model applied a roughly 138% FPL Medicaid cutoff to the OLDER_CHILD category and diverted higher-income children to CHIP. North Carolina’s applicable older-child Medicaid limit admits Child 3 at the traced MAGI of 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model converted its asserted 207% FPL limit into an incorrect $56,232 ceiling and compared that with its own MAGI estimate. The applicable older-child limit admits the traced 2.13-times-FPL MAGI, and employer-sponsored coverage does not negate eligibility."
+us,scenario_026,child3_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,The model used an approximate 211% FPL cutoff and an erroneous dollar threshold to exclude the child. The engine’s applicable OLDER_CHILD limit includes the traced MAGI of 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model applied the obsolete or inapplicable 133% plus 5% disregard framework to a nine-year-old and then treated CHIP as the only higher-income route. The applicable North Carolina OLDER_CHILD Medicaid threshold admits 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model overstated the household’s income level as about 260% FPL. The Medicaid MAGI computation yields 2.13 times FPL, which falls within the applicable OLDER_CHILD limit."
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model incorrectly treated employer-sponsored insurance as a bar to Medicaid eligibility and also imposed an income ceiling below the applicable older-child limit. Child 3 qualifies in the OLDER_CHILD category at 2.13 times FPL regardless of existing employer-sponsored coverage.
+us,scenario_026,child3_medicaid_eligible,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium by adding it to annual gross wages, even though the prompt states gross wages include overtime. It then applied an inapplicable 138% FPL older-child cutoff instead of the limit that admits the traced 2.13-times-FPL MAGI."
+us,scenario_026,child3_medicaid_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,The model imposed a 133% FPL ceiling on North Carolina’s older-child Medicaid category. The applicable category limit admits Child 3 at 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 133% FPL cutoff that does not govern the applicable 2026 OLDER_CHILD determination. At the traced Medicaid MAGI of 2.13 times FPL, Child 3 satisfies the category’s income test."
+us,scenario_026,child3_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer implies an income ceiling below the household’s traced 2.13-times-FPL MAGI. North Carolina’s applicable OLDER_CHILD Medicaid limit includes that income level.
+us,scenario_026,child3_medicaid_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model substituted a 133% FPL Medicaid limit and assigned the child to CHIP above it. The applicable OLDER_CHILD Medicaid threshold admits the traced 2.13-times-FPL MAGI.
+us,scenario_026,child3_medicaid_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,The model applied a 133% FPL ceiling to the older-child determination. The applicable North Carolina threshold includes Child 3 at the engine-computed 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The answer assumes the North Carolina child Medicaid income limit is below 2.13 times FPL. The applicable OLDER_CHILD limit admits that traced MAGI level.
+us,scenario_026,child3_medicaid_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,The answer applies an income limit below the traced MAGI without using the applicable OLDER_CHILD threshold. Child 3’s 2.13-times-FPL MAGI passes that category’s income test.
+us,scenario_026,child3_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated 133% FPL as the Medicaid ceiling and 210% FPL as a separate CHIP ceiling, excluding the child at about 213% FPL. The applicable OLDER_CHILD Medicaid limit admits 2.13 times FPL, and employer-sponsored insurance is not a Medicaid eligibility bar."
+us,scenario_026,child3_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,The model used an unspecified benchmark Medicaid threshold below the household’s income. The applicable North Carolina OLDER_CHILD threshold admits the traced MAGI of 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model required an unstated special status instead of applying the ordinary OLDER_CHILD MAGI pathway. Child 3 qualifies through that pathway because the traced 2.13-times-FPL MAGI is within its income limit.
+us,scenario_026,child3_medicaid_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model compared earnings with an incorrectly low North Carolina child Medicaid limit. Under the applicable OLDER_CHILD threshold, the engine-calculated MAGI of 2.13 times FPL is qualifying."
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model estimated a North Carolina child Medicaid ceiling below the household’s income instead of applying the applicable OLDER_CHILD parameter. That parameter admits the traced MAGI of 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,The model applied an incorrect Medicaid income limit for a nine-year-old. Child 3’s traced MAGI of 2.13 times FPL is within North Carolina’s applicable OLDER_CHILD limit.
+us,scenario_026,child3_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The answer assumes the North Carolina child Medicaid ceiling falls below 2.13 times FPL. The applicable OLDER_CHILD threshold includes that income level.
+us,scenario_026,child3_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model applied an incorrectly low income threshold and treated employer-sponsored insurance as supporting ineligibility. Child 3 qualifies under the OLDER_CHILD income test at 2.13 times FPL, and existing employer coverage does not bar Medicaid eligibility."
+us,scenario_026,child3_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,The model imposed a 133% FPL limit on the age 6–18 Medicaid category. North Carolina’s applicable OLDER_CHILD limit admits the traced 2.13-times-FPL MAGI.
+us,scenario_026,child3_medicaid_eligible,grok-4.6,llm_error,thresholds_rates,False,The model used a 133% FPL ceiling for children ages 6–18 and compared it with a separately miscomputed MAGI. The governing OLDER_CHILD limit includes the engine-calculated MAGI of 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated the income level as about 240% FPL and applied a 133% FPL base limit. The Medicaid MAGI is 2.13 times FPL, and the applicable OLDER_CHILD threshold admits it."
+us,scenario_026,child3_medicaid_eligible,inkling,llm_error,categorical_eligibility,False,"The model imposed a 133% FPL limit and treated employer-sponsored coverage as adverse to eligibility. The applicable OLDER_CHILD threshold admits 2.13 times FPL, and existing employer coverage does not disqualify Child 3."
+us,scenario_026,child3_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for child3_medicaid_eligible, so it failed the required output contract."
+us,scenario_026,child3_medicaid_eligible,kimi-k3,llm_error,thresholds_rates,False,The model applied a 133% or 138% FPL ceiling to the OLDER_CHILD category and also overstated the household’s income level as about 226% FPL. The traced Medicaid MAGI is 2.13 times FPL and is within the applicable category limit.
+us,scenario_026,child3_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,The model assumed the North Carolina Medicaid-for-children income limit was below the household’s income. The applicable OLDER_CHILD threshold admits the traced 2.13-times-FPL MAGI.
+us,scenario_026,child3_medicaid_eligible,ox-alpha,llm_error,thresholds_rates,False,The model used a 133% FPL ceiling for the age 6–18 group. The applicable North Carolina OLDER_CHILD limit includes Child 3 at 2.13 times FPL.
+us,scenario_026,child3_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied a 133% FPL Medicaid cutoff and used an erroneous five-person federal poverty guideline to construct its dollar ceiling. The applicable OLDER_CHILD limit admits the correctly computed MAGI of 2.13 times FPL, and employer-sponsored insurance does not bar eligibility."
+us,scenario_026,child3_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The answer assumes the child Medicaid limit is below the traced income level. North Carolina’s applicable OLDER_CHILD threshold admits Child 3 at 2.13 times FPL.
 us,scenario_026,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model omitted the qualified-overtime deduction and inconsistently restricted the three-child credit to $1,500. The $6,600 total CTC supplies $4,586.51 of nonrefundable credit to eliminate the entire tentative tax."
 us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model explicitly derived that the child tax credit eliminates the tentative liability, then submitted $4,107 instead of zero. It also omitted the qualified-overtime deduction, but even its own tentative-tax calculation was fully offset by the available CTC."
 us,scenario_026,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated only $605 of the child tax credit as nonrefundable by subtracting refundable portions from the credit before applying it to tax. The credit is first usable against liability up to $4,586.51, eliminating the tax; only the remaining credit is considered under refundable-credit rules."
@@ -1522,33 +1602,35 @@ us,scenario_026,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,
 us,scenario_026,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the overtime premium by adding it to wages already stated to include overtime, then omitted the qualified-overtime deduction and applied obsolete personal exemptions and a $1,000-per-child CTC. Gross employment income remains $85,208.82, the overtime deduction reduces taxable income to $42,354.26, and the resulting tax is fully offset by CTC."
 us,scenario_026,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not be parsed or evaluated as a substantive calculation."
 us,scenario_026,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model confused the former personal-exemption amount with the married-joint standard deduction, failed to subtract the pre-tax 401(k), IRA, and qualified-overtime deductions, and used an incorrect $3,700 CTC. The applicable deductions yield $42,354.26 of taxable income, and $4,586.51 of the $6,600 CTC eliminates the tentative tax."
-us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that pre-credit tax absorbs the entire $6,600 CTC. The correct nonrefundable-versus-refundable allocation leaves $2,013.49 as refundable CTC."
-us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model assigned an unsupported approximate refund of $1,000 per child instead of calculating the CTC remaining after the nonrefundable portion offsets tax. That allocation yields $2,013.49, not $3,000."
-us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used a $6,000 total CTC and incorrectly treated all of it as absorbed by tax liability. Applying the 2026 CTC amount and allocating it between nonrefundable and refundable portions produces $2,013.49 of refundable CTC."
-us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly concluded that tax liability consumes the full child credit. The 2026 CTC allocation leaves $2,013.49 refundable after the nonrefundable portion is used."
-us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model recognized that the unused CTC is refundable but overstated pre-credit tax at approximately $5,450, leaving only $1,150. The engine's tax and nonrefundable-credit computation leaves $2,013.49 of the CTC refundable."
-us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an approximate $6,000 CTC, standard deduction, and bracket calculation, producing an overstated $5,815 tax amount and only $185 of unused credit. The applicable 2026 parameters and exact tax calculation leave $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium on top of annual gross wages and treated Roth contributions as reducing AGI. That inflated income and tax liability, leading it to incorrectly conclude that the entire CTC was nonrefundable rather than leaving $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model incorrectly treated the nonrefundable CTC as fully exhausting the child credit. After the credit offsets tax, $2,013.49 remains payable as refundable CTC."
-us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied obsolete reversion parameters—a $3,000 total CTC, $1,000 refundable cap per child, and a $3,000 earned-income threshold. The applicable 2026 CTC calculation produces $2,013.49 of refundable credit."
-us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The answer omitted the refundable CTC solely because of the household's income level. Income eliminates EITC but does not eliminate the refundable CTC, which equals $2,013.49."
-us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $3,000 Child Tax Credit and concluded it was completely absorbed by tax. Under the applicable 2026 CTC parameters, the nonrefundable allocation leaves $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $61,093 without support from the listed deductible inputs and consequently awarded a spurious $930 EITC. It also understated refundable CTC at $926; the correct components are zero EITC and $2,013.49 refundable CTC."
-us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The answer's zero implies that it omitted the refundable portion of the Child Tax Credit. The household receives $2,013.49 of refundable CTC even though its other refundable credits are zero."
-us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model correctly eliminated EITC but incorrectly asserted that the nonrefundable CTC covers the entire child credit. The allocation leaves $2,013.49 of refundable CTC."
-us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC and calculated $5,676.12 of tax liability, leaving only $323.88. The applicable 2026 credit parameters and exact tax computation leave $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the children's being older than six as eliminating the refundable child credit. All three children are under 17 and qualify for the CTC, whose refundable portion is $2,013.49."
-us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model used income level as a blanket reason to deny every refundable credit. Although income eliminates EITC, the household still has $2,013.49 of refundable CTC."
-us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied a post-2025 regime with only $1,000 nonrefundable child credits. The applicable 2026 CTC rules yield an unused refundable portion of $2,013.49."
-us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model identified the correct refundable-credit pathway but miscomputed the CTC remaining after the nonrefundable offset by $45.27. The exact allocation yields $2,013.49, not $2,058.76."
-us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated this income as disqualifying the household from refundable CTC. Income eliminates EITC, while the refundable CTC remains $2,013.49."
-us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC and concluded tax liability exceeded it. Applying the applicable 2026 child-credit amount and allocation leaves $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,period_annualization,False,"The model added the separately reported $7,746 FLSA overtime premium to the $85,209 annual gross-wage total even though gross wages already include overtime. It also used a $3,000 CTC, and these errors led it to miss the $2,013.49 refundable portion."
-us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no parseable value or explanation for federal_refundable_credits.
-us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 total CTC but overstated the amount absorbed nonrefundably as $5,873. The exact tax and credit allocation leaves $2,013.49 refundable, not $727."
-us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly set EITC to zero but also set Additional CTC to zero without allocating the unused child credit. That allocation produces $2,013.49 of refundable CTC."
-us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC and an overstated $5,479 nonrefundable offset, leaving only $521. The applicable 2026 CTC parameters and exact allocation leave $2,013.49 refundable."
-us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model treated $4,300 as refundable based on the number of qualifying children without subtracting the CTC used nonrefundably against tax. The remaining refundable CTC after that offset is $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction and consequently overstated pre-credit income tax as about $5,500. The correct tax calculation uses $4,586.51 of the $6,600 CTC nonrefundably, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model assigned an unsupported approximate $1,000 refundable amount to each child instead of subtracting the nonrefundable CTC used from the $6,600 total credit. That computation leaves a refundable CTC of $2,013.49, not $3,000."
+us,scenario_026,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction and used an overstated tax liability of about $5,600, then incorrectly treated the entire CTC as absorbed. Only $4,586.51 is used nonrefundably, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to apply the qualified-overtime deduction and therefore concluded that regular tax absorbed the entire CTC. The resulting refundable balance is $6,600 minus $4,586.51, or $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model correctly used a $6,600 total CTC but overstated pre-credit tax as roughly $5,450 by omitting the qualified-overtime deduction. Pre-credit tax available to absorb the CTC is $4,586.51, so the refundable remainder is $2,013.49."
+us,scenario_026,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $7,746 qualified-overtime deduction and computed tax on about $52,431 of taxable income. Applying that deduction produces $4,586.51 of tax absorbed by the nonrefundable CTC and a $2,013.49 refundable remainder."
+us,scenario_026,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium by adding it to annual gross wages even though the prompt states that gross wages already include overtime, and it also failed to deduct qualified overtime. This inflated income and tax enough to erase the refundable CTC that equals $2,013.49."
+us,scenario_026,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model asserted that the nonrefundable CTC fully offsets tax without applying the qualified-overtime deduction. Tax absorbs only $4,586.51 of the $6,600 CTC, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly reverted the 2026 CTC to $1,000 per child and a $3,000 household total. The applicable total CTC is $6,600; after $4,586.51 is used nonrefundably, $2,013.49 remains refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The zero answer skips the refundable CTC computation for three qualifying children. The $6,600 total CTC exceeds the $4,586.51 usable nonrefundable amount, yielding $2,013.49 of refundable credit."
+us,scenario_026,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC under obsolete parameters and therefore concluded that tax absorbed it completely. The 2026 total is $6,600, of which $4,586.51 is used nonrefundably and $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model invented an AGI of $61,093 and consequently awarded a $930 EITC even though this household's income fully phases out EITC. It also understated the refundable CTC as $926; the sole refundable credit is the $2,013.49 refundable CTC."
+us,scenario_026,federal_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The zero answer omits the unused portion of the CTC. Of the $6,600 credit, only $4,586.51 is used nonrefundably, so $2,013.49 is refundable."
+us,scenario_026,federal_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that the nonrefundable CTC covers all available credit by failing to account for the qualified-overtime deduction in the tax calculation. Tax uses $4,586.51 of the $6,600 CTC, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used the correct residual-credit framework but overstated pre-credit tax as $5,676.12 because it omitted the qualified-overtime deduction. The nonrefundable amount used is $4,586.51, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model wrongly treated the children's being over age six as eliminating the refundable CTC; all three children are under 17 and qualify. Their $6,600 CTC exceeds the $4,586.51 nonrefundable usage by $2,013.49."
+us,scenario_026,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred that this income level precluded every refundable credit, conflating the EITC income limit with refundable CTC eligibility. EITC is zero, but three qualifying children generate a $2,013.49 refundable CTC balance."
+us,scenario_026,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied obsolete post-2025 rules under which each $1,000 child credit was nonrefundable. The applicable 2026 CTC totals $6,600 and permits a $2,013.49 refundable portion after nonrefundable usage."
+us,scenario_026,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model recognized the correct $6,600 CTC and residual-credit method but understated the nonrefundable amount used by $45.27. The correct usage is $4,586.51, leaving $2,013.49 rather than $2,058.76 refundable."
+us,scenario_026,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated income above the EITC limit as also disqualifying the household from refundable CTC. The children qualify for the CTC, and $2,013.49 remains refundable after the credit offsets tax."
+us,scenario_026,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $3,000 total CTC and therefore concluded that tax exceeded the credit. The applicable $6,600 CTC exceeds the $4,586.51 nonrefundable usage, producing a $2,013.49 refund."
+us,scenario_026,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 CTC but overstated the nonrefundable amount used as $4,671. Applying the qualified-overtime deduction yields $4,586.51 of usage and a $2,013.49 refundable balance."
+us,scenario_026,federal_refundable_credits,grok-build-0.1,llm_error,period_annualization,False,"The model double-counted the overtime premium by raising earned income to $90,047 even though the $85,209 annual wages already include overtime, and it used an obsolete $3,000 CTC. The applicable $6,600 CTC leaves $2,013.49 refundable after $4,586.51 is used against tax."
+us,scenario_026,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so its response could not be parsed or evaluated as a substantive estimate."
+us,scenario_026,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the correct $6,600 CTC but overstated tax absorbed by the nonrefundable credit as $5,873 because it omitted the qualified-overtime deduction. Only $4,586.51 is absorbed, leaving $2,013.49 refundable."
+us,scenario_026,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model correctly eliminated EITC but incorrectly set the additional CTC to zero solely from the household's income level. The CTC is not phased out at this income, and $2,013.49 remains after its nonrefundable use."
+us,scenario_026,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model correctly used a $6,600 CTC but overstated pre-credit tax as $5,586.60 because it omitted the qualified-overtime deduction. The correct nonrefundable usage is $4,586.51, producing a $2,013.49 refundable remainder."
+us,scenario_026,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a $6,000 total CTC instead of the applicable $6,600 and also overstated tax available to absorb it. The correct calculation uses $4,586.51 nonrefundably and refunds the remaining $2,013.49."
+us,scenario_026,federal_refundable_credits,qwen3.8-max,llm_error,other,False,"The model treated $4,300 as the refundable Additional CTC without subtracting the CTC used against income-tax liability. The $6,600 total credit first offsets $4,586.51 of tax, leaving $2,013.49 refundable."
 us,scenario_026,free_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model reversed the income comparison by calling $85,208.82 well below the free-meal standard. That income is 220% of the family-of-five poverty guideline, and without SNAP/TANF categorical eligibility or universal free meals in North Carolina, the household must pay full price."
 us,scenario_026,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_026,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -1565,30 +1647,32 @@ us,scenario_026,reduced_price_school_meals_eligible,gemini-3.5-flash,llm_error,t
 us,scenario_026,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_026,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_026,spouse_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model allowed only $1,500 of child deductions instead of the applicable $4,500, leaving taxable income $3,000 too high. Its submitted $2,882 also contradicts its own recomputed tax of about $2,207."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly started from federal taxable income, substituted a federal standard deduction, used a 4.75% rate, and invented a $2,716 North Carolina dependent credit instead of applying North Carolina's $25,500 standard deduction and $4,500 child deduction to federal AGI. Its final $5,238 is also disconnected from every intermediate amount in its reasoning."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $130 traditional IRA deduction from AGI and overstated the child deduction as $7,500 rather than $4,500. It then submitted $3,041 despite calculating liabilities between $1,972 and $2,101, so its final value does not follow its own computation."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $130 deductible traditional IRA contribution and the $4,500 North Carolina child deduction. Those omissions inflated taxable income from $52,300.52 to about $56,931 and produced the excessive tax."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $21,500 joint standard deduction and a $9,000 child deduction instead of the applicable $25,500 and $4,500 deductions. The net $500 over-deduction reduced taxable income below $52,300.52 and understated the tax."
-us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 FLSA overtime premium even though gross wages already include all overtime, omitted the $4,500 child deduction, and used 4.25% instead of the 2026 rate schedule. Its asserted retirement-contribution addback and jump from $2,743 to $3,350 have no applicable North Carolina computation behind them."
-us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 North Carolina child deduction. It therefore taxed $56,801 rather than $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $21,208 employer-sponsored insurance premium from wages even though the trace's federal AGI is $82,300.52, and it overstated the child deduction as $6,000 instead of $4,500. These errors drove taxable income down to $29,593 rather than $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,538 implies substantially more taxable income than the correct $52,300.52 at North Carolina's rate. The generic explanation never applies the $25,500 joint standard deduction and $4,500 child deduction that determine the liability."
-us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI by the $21,208 employer-sponsored insurance premium and used a $6,000 child deduction instead of $4,500. It consequently taxed $29,593 rather than $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model invented a second $130 traditional IRA deduction for the head, omitted the $4,500 child deduction, and used a 3.74% rate. North Carolina instead taxes $52,300.52 under the 2026 rate schedule."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated deductions or credits as eliminating the liability, but the $25,500 standard deduction and $4,500 child deduction leave $52,300.52 taxable. That positive taxable income produces $2,086.79 before refundable credits, not zero."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $4,875 estimate does not apply the North Carolina deductions and rate schedule to the traced AGI. Federal AGI of $82,300.52 falls to $52,300.52 after the $25,500 standard deduction and $4,500 child deduction, yielding $2,086.79."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model understated the taxable-income base while naming only the standard deduction and no child-deduction calculation. The required deductions leave $52,300.52 taxable, whose scheduled tax is $2,086.79 rather than $1,957.29."
-us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $2,027 reflects an understated taxable-income base after the joint and child deductions. The applicable deductions total $30,000, leaving $52,300.52 and a liability of $2,086.79."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied a rough flat-rate calculation to approximately $85,000 of wages instead of starting from $82,300.52 of federal AGI and subtracting $30,000 of North Carolina deductions. The correct taxable-income base is $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 North Carolina child deduction. It taxed about $56,801 rather than $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium even though annual gross wages already include overtime, omitted the $4,500 child deduction, and used 4.25%. These errors produced $64,547 of taxable income instead of $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $74,555 without a valid listed adjustment and overstated the child deduction as $7,500 rather than $4,500. The correct computation begins at $82,300.52 and leaves $52,300.52 taxable."
-us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
-us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used gross wages as AGI, omitting the $2,778 traditional 401(k) contribution and $130 traditional IRA deduction, and it also omitted the $4,500 child deduction. The correct AGI is $82,300.52 and the correct taxable income is $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated standard deductions and personal exemptions as reducing North Carolina taxable income to approximately zero. The applicable deductions total $30,000, leaving $52,300.52 taxable and a positive $2,086.79 liability."
-us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $7,746 overtime premium already included in annual wages, omitted the pre-tax retirement adjustments and $4,500 child deduction, and used a 4.5% rate. Those mistakes inflated taxable income to $67,455 instead of $52,300.52."
-us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model used an unsupported $81,509 taxable-income figure, a 4.5% rate, and an invented $19.41 nonrefundable credit. North Carolina starts from $82,300.52 of federal AGI, subtracts $30,000 of state deductions, and taxes the resulting $52,300.52 under the 2026 schedule."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model allowed only $500 per child instead of the applicable $1,500 per child, overstating taxable income by $3,000. Its submitted $2,882 also contradicts its own recomputed tax of about $2,207, adding a separate final-answer computation failure."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly started from federal taxable income, used a federal-style $29,200 deduction and a nonexistent $2,716 North Carolina dependent credit, then submitted a number unsupported by any calculation it presented. North Carolina taxable income is $82,300.52 minus the $25,500 state standard deduction and $4,500 child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used a $7,500 child deduction instead of $4,500 and alternated between incorrect 4.25% and correct 3.99% rates. Its $3,041 submission follows neither of its calculated liabilities and disregards the resulting $52,300.52 taxable-income base."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 North Carolina child deduction entirely and also used AGI of about $82,431 rather than $82,300.52. Applying 3.99% after both the $25,500 standard deduction and child deduction yields $2,086.79."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an outdated $21,500 joint standard deduction and overstated the child deduction as $3,000 per child. The applicable deductions are $25,500 plus $1,500 for each of three children, producing taxable income of $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium by adding $7,746 to annual gross wages, even though gross wages already include overtime, and it omitted the $4,500 child deduction. It also used 4.25% instead of the 2026 rate of 3.99% and invented an unsupported retirement-contribution addback."
+us,scenario_026,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 North Carolina child deduction, leaving taxable income at $56,800.52 rather than $52,300.52. That omission overstates the tax by $179.55."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the separately listed $21,208 employer-sponsored insurance premium from wages even though the benchmark's wage amount and engine AGI do not authorize that additional exclusion. It then used a $6,000 child deduction rather than $4,500, driving taxable income far below $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $3,538 implies a taxable base of about $88,672 at the 3.99% rate, exceeding even the household's gross wages. It therefore failed to apply the engine-derived AGI of $82,300.52 together with the $25,500 standard deduction and $4,500 child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model used an erroneously reduced federal AGI of $61,093, reflecting an unauthorized subtraction of the $21,208 insurance-premium input from wages. It also allowed $2,000 per child instead of $1,500, rather than taxing $52,300.52 at 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied an incorrect 3.74% rate instead of North Carolina's 3.99% rate for 2026 and invented a second $130 IRA deduction for the head, although only the spouse has that contribution listed. It also omitted the $4,500 child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions and credits as eliminating the liability. The specified deductions reduce AGI only to $52,300.52 of taxable income, which remains subject to the 3.99% tax."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $4,875 estimate does not reflect the $25,500 joint standard deduction or the $4,500 child deduction and exceeds the tax generated by applying the relevant rate even to gross wages. The correct taxable base is $52,300.52, not an unspecified wage-level estimate."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $1,957.29 answer corresponds to taxing approximately $49,055 at 3.99%, understating the engine-derived taxable income of $52,300.52 by $3,245.52. The model therefore applied an unsupported AGI reduction or excessive deduction beyond the $25,500 standard and $4,500 child deductions."
+us,scenario_026,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"Although the model named the correct rate and deduction categories, its $2,027 answer corresponds to taxable income of about $50,802 rather than $52,300.52. It overstated the combined deductions by about $1,499."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an unspecified approximation based on gross wages and did not calculate the engine-derived AGI or the full North Carolina deductions. It needed to subtract both the $25,500 joint standard deduction and $4,500 child deduction from $82,300.52 before applying 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 child deduction and taxed $56,801 instead of $52,300.52. Its rate and standard deduction were otherwise aligned with the required computation."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used an unsupported federal AGI of $74,555 rather than $82,300.52. After the $25,500 standard deduction and $4,500 child deduction, the correct taxable base is $52,300.52, not $49,055."
+us,scenario_026,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,payroll_tax_base,False,"The model double-counted the $7,746 overtime premium by adding it to annual gross wages, despite the instruction that gross wages already include overtime, and omitted the $4,500 child deduction. It also applied 4.25% instead of the 2026 rate of 3.99%."
+us,scenario_026,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an unsupported AGI of $74,555 and overstated the child deduction as $7,500 rather than $4,500. These errors reduced taxable income to $41,555 instead of $52,300.52."
+us,scenario_026,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required numeric result was missing."
+us,scenario_026,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used gross wages of $85,209 as AGI, failing to subtract the deductible traditional 401(k) and IRA contributions that produce AGI of $82,300.52. It also omitted the $4,500 North Carolina child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated standard deductions and personal exemptions as reducing taxable income to approximately zero. The available deductions total $30,000, leaving $52,300.52 subject to tax; North Carolina does not provide enough personal exemptions here to erase that base."
+us,scenario_026,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used an unsupported AGI of $74,555 rather than $82,300.52. Applying the $25,500 standard deduction and $4,500 child deduction to the correct AGI leaves $52,300.52, not $49,055."
+us,scenario_026,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $7,746 overtime premium even though annual gross wages already include overtime, omitted the retirement deductions and the $4,500 child deduction, and used 4.5% instead of 3.99%. Its discussion of an estimated child credit did not substitute for the applicable child deduction."
+us,scenario_026,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable income of $81,509, a federal-standard-deduction framework, an incorrect 4.5% rate, and an invented $19.41 nonrefundable credit. North Carolina instead taxes $52,300.52 after its $25,500 standard deduction and $4,500 child deduction at 3.99%."
 us,scenario_026,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_027,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output, violating the required structured-output contract."
 us,scenario_027,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model correctly derived zero taxable income, zero gross tax, and no nonrefundable credits, but submitted -3195 instead of the zero produced by its calculation. Federal income tax before refundable credits does not become negative here, and its explanation explicitly ends with a value inconsistent with its own derivation."
@@ -1606,48 +1690,51 @@ us,scenario_027,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F
 us,scenario_027,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_027,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_027,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated failure of the ordinary net-income test as a complete disqualification. Connecticut categorical eligibility preserves eligibility here, so the negative formula amount is replaced by the monthly minimum allotment."
-us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied the federal elderly/disabled resource ceiling to the $48,000 bank balance even though Connecticut categorical eligibility removes that resource test for this household. It also incorrectly characterized $2,513 of monthly SSDI as exceeding the gross-income threshold it quoted as about $2,900."
-us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model disqualified the household under an approximately $4,500 resource limit. Connecticut categorical eligibility eliminates that asset-based disqualification, leaving the household entitled to the minimum allotment."
-us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model applied the ordinary elderly/disabled SNAP resource ceiling to the household's bank assets. Connecticut categorical eligibility bypasses that resource test, and the eligible two-person household receives the minimum allotment when its formula benefit is negative."
-us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated countable income above the ordinary net-income limit as eliminating SNAP entirely. Categorical eligibility preserves eligibility, and the two-person minimum allotment applies after the standard formula produces a negative amount."
-us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model correctly identified elderly/disabled deductions but then imposed a $4,500 asset ceiling. Connecticut categorical eligibility removes that resource barrier, so $48,000 in bank assets does not prevent the minimum allotment."
-us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model disqualified the household solely under a $4,250 resource limit. Connecticut categorical eligibility makes that limit inapplicable and permits the monthly minimum benefit."
-us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model converted net income above 100% FPL into zero eligibility and stopped. Categorical eligibility keeps the household eligible, and the negative benefit formula is floored at the minimum allotment for a two-person household."
-us,scenario_027,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model correctly found that the ordinary maximum-allotment-minus-contribution formula reaches zero, but failed to apply the minimum allotment for an eligible one- or two-person household. That floor produces positive benefits in every month."
-us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model applied a generic SNAP resource limit to the $48,000 bank balance. Connecticut categorical eligibility removes the asset test applicable under the ordinary federal pathway."
-us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model recognized categorical eligibility but incorrectly set the award to zero when 30% of net income exceeded the maximum allotment. An eligible two-person household receives the statutory monthly minimum instead.
-us,scenario_027,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model applied a flat $23 monthly minimum for all twelve months. PolicyEngine uses $23.84 in earlier months and $24.37 in later months, whose monthly aggregation totals $287.68 rather than $276."
-us,scenario_027,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model correctly used categorical eligibility and the minimum-allotment rule but annualized an obsolete rounded $23 amount uniformly. The applicable monthly minimum changes from $23.84 to $24.37 during the year, totaling $287.68."
-us,scenario_027,snap,gemini-3.7-flash,llm_error,period_annualization,False,The model used $23 per month throughout the year without applying the 2026 monthly parameter values. Aggregating $23.84 in earlier months and $24.37 in later months yields $287.68.
-us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, violating the required structured-output contract."
-us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,"The model relied on generic asset and income rules and treated $48,000 of bank assets as disqualifying. Connecticut categorical eligibility removes the resource test and preserves the minimum-allotment entitlement."
-us,scenario_027,snap,gpt-5.4-nano,llm_error,other,False,"The model disregarded the instruction that unlisted numeric facts are zero and treated shelter and income inputs as missing. With those inputs fixed at zero, categorical eligibility and the two-person minimum-allotment rule produce $287.68 annually."
-us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model stopped after finding that net income exceeded the ordinary limit or benefit phaseout. Categorical eligibility keeps the household eligible, and the minimum allotment replaces the negative formula result."
-us,scenario_027,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model equated a nonpositive standard benefit calculation with no SNAP award. Because the household remains categorically eligible, the one- or two-person minimum allotment applies each month."
-us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model treated net income above the ordinary two-person limit as dispositive. Connecticut categorical eligibility preserves eligibility, and the negative formula benefit is floored at the monthly minimum."
-us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model imposed the ordinary elderly/disabled household resource ceiling on the $48,000 bank balance. Connecticut categorical eligibility removes that resource test for this household."
-us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model used generic SNAP asset limits to disqualify the household and did not apply Connecticut categorical eligibility. The assets therefore do not eliminate eligibility, and the minimum allotment is payable."
-us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model treated an approximately $4,500 resource limit as controlling regardless of categorical eligibility. Connecticut categorical eligibility bypasses that limit and leaves the household eligible for the monthly minimum."
-us,scenario_027,snap,grok-build-0.1,llm_error,period_annualization,False,"The model compared annual net income directly with an annual maximum allotment and concluded that the benefit was zero. SNAP is computed monthly, and after categorical eligibility preserves participation, each negative monthly formula result is replaced by the minimum allotment."
-us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model treated net income above 100% FPL as a complete eligibility failure. Connecticut categorical eligibility keeps the elderly/disabled household eligible, triggering the minimum allotment despite the negative ordinary formula result."
-us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model correctly noted the absence of shelter deductions but incorrectly made the ordinary net-income limit dispositive. Categorical eligibility preserves eligibility and requires the two-person monthly minimum allotment.
-us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model stopped when adjusted monthly income remained above the ordinary net-income limit. Under Connecticut categorical eligibility, that does not reduce the award below the minimum allotment for an eligible two-person household."
-us,scenario_027,snap,minimax-m3,llm_error,asset_resource,False,"The model used incorrect gross-limit arithmetic and separately imposed a $4,250 asset ceiling. The household passes the traced gross test at 1.44 times FPL, and Connecticut categorical eligibility removes the resource-based disqualification."
-us,scenario_027,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model incorrectly said $2,536 monthly income exceeded the applicable gross threshold and also imposed an ordinary resource ceiling on $48,000 of assets. The household passes the gross test and categorical eligibility removes the asset test, leaving the minimum allotment payable."
-us,scenario_027,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly placed net income below the poverty guideline and therefore calculated a large positive formula benefit. The applicable deductions leave monthly net income around $2,113–$2,118, making the ordinary formula negative and limiting the household to $287.68 of annual minimum allotments."
+us,scenario_027,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model applied the ordinary 100%-FPL net-income eligibility test after acknowledging Connecticut BBCE. TANF non-cash categorical eligibility keeps this household eligible despite that test, and the negative formula benefit is replaced by the monthly minimum allotment."
+us,scenario_027,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model imposed ordinary gross-income and resource limits, including an elderly/disabled asset ceiling, despite Connecticut TANF non-cash categorical eligibility. The $48,000 bank balance does not disqualify this categorically eligible household, which receives the minimum allotment."
+us,scenario_027,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model disqualified the household solely under an ordinary elderly/disabled SNAP resource limit. Connecticut TANF non-cash categorical eligibility removes that resource-test barrier, so the household proceeds to the benefit calculation and receives the minimum allotment."
+us,scenario_027,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model applied an approximately $4,500 resource limit instead of the Connecticut categorical-eligibility pathway. The $48,000 account therefore does not eliminate eligibility, and the minimum allotment applies."
+us,scenario_027,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated net income above the ordinary limit as dispositive. TANF non-cash categorical eligibility preserves SNAP eligibility, after which the negative standard formula is floored at the monthly minimum allotment."
+us,scenario_027,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model correctly identified the elderly/disabled deductions but then imposed an estimated $4,500 asset ceiling. Connecticut TANF non-cash categorical eligibility prevents the $48,000 bank account from disqualifying the household, so the minimum benefit is payable."
+us,scenario_027,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model used a $4,250 elderly/disabled resource limit as an automatic bar. It omitted Connecticut's TANF non-cash categorical eligibility, under which the household remains eligible and receives the minimum allotment."
+us,scenario_027,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model converted net income above 100% FPL into complete ineligibility. Connecticut categorical eligibility keeps the household eligible, and a negative allotment formula result triggers the minimum monthly benefit rather than zero."
+us,scenario_027,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly recognized Connecticut BBCE and a negative max-allotment-minus-contribution result but stopped at zero. An eligible one- or two-person household receives the SNAP minimum allotment, producing $287.68 over the year's monthly parameter periods."
+us,scenario_027,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model applied a general SNAP resource limit to the $48,000 account. Connecticut TANF non-cash categorical eligibility bypasses that ordinary resource test, leaving the household eligible for the minimum allotment."
+us,scenario_027,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model acknowledged categorical eligibility but treated 30% of net income exceeding the maximum allotment as producing no award. For an eligible two-person household, the minimum allotment replaces the negative formula result."
+us,scenario_027,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model found the correct minimum-allotment pathway but annualized a stale flat $23 monthly amount. PolicyEngine applies $23.84 in most months and $24.37 in later months, whose annual sum is $287.68 rather than $276."
+us,scenario_027,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model correctly applied BBCE and the minimum allotment but used $23 for every month. The 2026 monthly minimums are $23.84 and then $24.37, so aggregating the monthly values yields $287.68."
+us,scenario_027,snap,gemini-3.7-flash,llm_error,period_annualization,False,"The model used an outdated flat $23 minimum allotment and multiplied it by 12. The applicable monthly minimum changes from $23.84 to $24.37 during the year, totaling $287.68."
+us,scenario_027,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required structured-output contract."
+us,scenario_027,snap,gpt-5.4-mini,llm_error,asset_resource,False,The model treated the bank assets and ordinary income rules as disqualifying without applying Connecticut TANF non-cash categorical eligibility. That pathway bypasses the ordinary resource barrier and leads to the minimum allotment.
+us,scenario_027,snap,gpt-5.4-nano,llm_error,other,False,"The prompt explicitly sets every unlisted numeric input, including shelter costs and other income, to zero, so no additional inputs were needed. Using those zero inputs with categorical eligibility and the minimum-allotment floor yields $287.68, not zero."
+us,scenario_027,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated net income above the ordinary eligibility limit or formula phaseout as eliminating SNAP. Connecticut TANF non-cash categorical eligibility preserves eligibility, and the two-person minimum allotment applies when the formula is negative."
+us,scenario_027,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model concluded that a negative benefit calculation produces no allotment. Because the household is categorically eligible, the negative result is replaced with the mandatory minimum allotment for a two-person household."
+us,scenario_027,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model used net income above the two-person limit as a complete eligibility bar. Connecticut categorical eligibility keeps the household eligible despite that ordinary test, so it receives the monthly minimum allotment."
+us,scenario_027,snap,gpt-5.6-terra,llm_error,asset_resource,False,"The model applied the ordinary elderly/disabled resource ceiling to the $48,000 bank balance. Connecticut TANF non-cash categorical eligibility bypasses that ceiling and leaves the household eligible for the minimum benefit."
+us,scenario_027,snap,grok-4.3,llm_error,asset_resource,False,"The model relied on typical asset limits and an ordinary income screen rather than Connecticut's categorical-eligibility rules. The bank balance does not disqualify this TANF non-cash categorically eligible household, and the minimum allotment applies."
+us,scenario_027,snap,grok-4.5,llm_error,asset_resource,False,"The model treated an approximately $4,500 resource limit as controlling regardless of other eligibility pathways. Connecticut TANF non-cash categorical eligibility bypasses the ordinary asset test, so the household receives the minimum allotment."
+us,scenario_027,snap,grok-4.6,llm_error,thresholds_rates,False,The model correctly determined that 30% of net income exceeds the maximum allotment but incorrectly returned zero. Eligible one- and two-person households receive the minimum allotment when that formula result is nonpositive.
+us,scenario_027,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model compared annual net income directly with the annual maximum allotment and treated the resulting negative formula amount as zero. The calculation requires a 30% expected contribution and, because categorical eligibility persists, application of the monthly minimum-allotment floor."
+us,scenario_027,snap,inkling,llm_error,categorical_eligibility,False,"The model applied the ordinary 100%-FPL net-income test as a disqualifier. Connecticut TANF non-cash categorical eligibility preserves eligibility despite that result, and the household receives the minimum monthly allotment."
+us,scenario_027,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly noted the absence of shelter deductions but incorrectly made the 100%-FPL net-income test dispositive. Categorical eligibility through TANF non-cash assistance keeps the household eligible, triggering the minimum allotment."
+us,scenario_027,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated income remaining above the ordinary net-income limit after deductions as complete ineligibility. Connecticut categorical eligibility overrides that ordinary barrier, and the negative benefit formula is floored at the minimum allotment."
+us,scenario_027,snap,minimax-m3,llm_error,asset_resource,False,"The model used both an incorrect gross-limit comparison and an ordinary $4,250 resource ceiling to deny SNAP. The traced gross-income ratio is 1.44 of the $1,762.50 guideline and passes Connecticut's applicable test, while TANF non-cash categorical eligibility prevents the $48,000 account from disqualifying the household."
+us,scenario_027,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model applied the ordinary net-income test and treated a negative allotment formula as zero. Connecticut categorical eligibility preserves eligibility, and the minimum-allotment floor produces a positive monthly benefit."
+us,scenario_027,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model incorrectly stated that $2,536 monthly income exceeds the applicable gross limit and separately imposed an ordinary resource ceiling. The traced gross-income ratio is 1.44 of FPL and passes, while Connecticut TANF non-cash categorical eligibility prevents the $48,000 balance from barring SNAP."
+us,scenario_027,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly placed net income below the poverty guideline and calculated a near-maximum annual benefit. After the standard and medical deductions, monthly net income is about $2,113–$2,118, making 30% of net income exceed the two-person maximum allotment; categorical eligibility then yields only the $287.68 annual minimum."
 us,scenario_027,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the head's Social Security income from household MAGI merely because it was nontaxable for income-tax purposes, reducing Medicaid MAGI to $275 instead of the engine's 1.41 times FPL. It also invoked a disability pathway without applying its categorical and financial requirements; the spouse falls into category NONE."
-us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness or disability as automatic Medicaid eligibility and tested only the spouse's individually listed income and assets. Connecticut requires qualification under a specific pathway with its applicable financial rules, and this spouse qualifies under none of them."
-us,scenario_027,spouse_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that $30,435 was below Connecticut's disabled-adult income limits without identifying or applying the governing HUSKY C threshold and related rules. Disability alone does not supply eligibility, and the spouse qualifies under no Medicaid category."
-us,scenario_027,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model inferred eligibility from disability, blindness, and a vaguely characterized 'modest' household income instead of applying a defined Connecticut Medicaid pathway. The spouse qualifies under no category, while the applicable MAGI measure is 1.41 times FPL and exceeds the MAGI limit."
-us,scenario_027,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model assumed that blind or disabled status placed the spouse into a disability-based category and then asserted that spousal-counted income was within its limit without performing that pathway's test. The spouse is assigned category NONE, so neither HUSKY D nor a blind/disabled pathway establishes eligibility."
-us,scenario_027,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model asserted that $48,000 was below Connecticut's HUSKY C couple asset limit and therefore treated the spouse as financially eligible for an ABD pathway. That resource conclusion and automatic ABD categorization are wrong; the spouse qualifies through no Medicaid pathway."
-us,scenario_027,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated disability status alone as sufficient for Medicaid. The spouse must qualify through a specific Connecticut eligibility pathway, and the engine assigns category NONE."
-us,scenario_027,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required submission contract."
-us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model stated that household MAGI was well below Connecticut's adult expansion limit, but the computed MAGI is 1.41 times FPL, above the applicable 138% FPL threshold. Disability does not create an alternative qualifying category here."
-us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security from Medicaid MAGI and reduced household MAGI to the $275 interest amount. The engine's Medicaid MAGI is 1.41 times FPL, above the expansion threshold, and the spouse qualifies through no non-MAGI category."
-us,scenario_027,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model equated being blind and disabled with SSI-related Medicaid eligibility and treated the spouse's lack of individually listed income as dispositive. The spouse receives no SSI and qualifies under no Medicaid category, so the SSI-related shortcut does not apply."
+us,scenario_027,spouse_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced Medicaid MAGI to $275 by excluding the household's Social Security income merely because it was nontaxable for federal income tax. The Medicaid MAGI computation yields 1.41 times FPL, above the applicable limit, and the spouse has no separate SSI-related eligibility category."
+us,scenario_027,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated blindness or disability as conferring Medicaid eligibility once it assigned the spouse zero individual income and disregarded the head's assets. Blindness and disability do not by themselves create eligibility; the spouse receives no SSI and qualifies through none of Connecticut's modeled pathways.
+us,scenario_027,spouse_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that $30,435 was below Connecticut's disabled-adult income limits without applying a specific qualifying pathway or its financial test. The spouse's category is NONE, while her MAGI equals 1.41 times FPL and exceeds the applicable MAGI threshold."
+us,scenario_027,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed that a blind or disabled spouse with subjectively modest household income qualifies through an aged, blind, or disabled pathway. The spouse receives no SSI and satisfies no modeled Medicaid category, while her 1.41-FPL MAGI also fails the MAGI pathway."
+us,scenario_027,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model conflated disability or blindness with eligibility for HUSKY D or a disability-based category and never established that the spouse met the relevant pathway's financial conditions. The spouse's Medicaid category is NONE, and her MAGI of 1.41 times FPL is above the MAGI limit."
+us,scenario_027,spouse_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated disabled-and-blind status as automatic ABD categorization and declared both income and assets within HUSKY C limits without applying the actual eligibility test. The spouse receives no SSI and qualifies through no Medicaid category; the model's additional assertion that $48,000 is below the relevant couple asset limit does not establish eligibility."
+us,scenario_027,spouse_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated disability status alone as sufficient for Medicaid. The spouse receives no SSI and qualifies through none of Connecticut's modeled pathways, and her MAGI is 1.41 times FPL."
+us,scenario_027,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for spouse_medicaid_eligible, violating the required output contract."
+us,scenario_027,spouse_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model characterized household MAGI as well below Connecticut's adult Medicaid limit. The computed Medicaid MAGI is 1.41 times FPL, above the applicable threshold, and disability does not supply another qualifying category."
+us,scenario_027,spouse_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly set Medicaid MAGI to $275 by excluding Social Security solely because it was excluded from ordinary taxable income. The Medicaid MAGI computation yields 1.41 times FPL, above the expansion threshold."
+us,scenario_027,spouse_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly used interest alone as household Medicaid MAGI and therefore placed the spouse below 138% FPL. The applicable computation produces MAGI of 1.41 times FPL, so HUSKY D income eligibility fails, and blindness or disability supplies no other category."
+us,scenario_027,spouse_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated blindness, disability, and zero personal income as sufficient for SSI-related Medicaid. The spouse receives zero SSI and does not qualify through an SSI-linked or other Medicaid category."
 us,scenario_027,spouse_medicare_eligible,claude-fable-5,llm_error,categorical_eligibility,False,The model treated disability status alone as sufficient for under-65 Medicare eligibility. It ignored that the spouse has no stated SSDI entitlement or completed 24-month waiting period and therefore does not qualify at age 39.
 us,scenario_027,spouse_medicare_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that disability status confers Medicare eligibility regardless of age. For an under-65 spouse, the listed disability must be accompanied by a qualifying Medicare entitlement pathway, such as the required SSDI entitlement history, which is absent."
 us,scenario_027,spouse_medicare_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model converted the spouse's disability flag into receipt of disability benefits. The facts list no SSDI for the spouse and no completed waiting period, so the under-65 Medicare disability pathway is not established."
@@ -1662,16 +1749,18 @@ us,scenario_027,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wron
 us,scenario_027,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_027,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_027,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an inapplicable approximately 314% FPL upper limit and concluded that income around $61,277 qualified. Under the applicable 2026 Pennsylvania CHIP threshold, that income is over the limit, and Child 1 has no Medicaid eligibility pathway."
-us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model used a 314% FPL CHIP ceiling instead of the applicable 2026 Pennsylvania income threshold. Child 1 passes the age test but fails the CHIP income test at approximately $61,277 and is not Medicaid-eligible."
-us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model treated 314% FPL as Pennsylvania’s operative CHIP income ceiling and also omitted child support from its stated $60,010 income figure. The trace uses household income of approximately $61,277, which exceeds the applicable CHIP limit, with no Medicaid pathway available."
-us,scenario_028,child1_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted that household income was within Pennsylvania’s CHIP limits without applying the applicable 2026 threshold. Approximately $61,277 exceeds that limit, while age 10 satisfies the age condition and Medicaid eligibility is false."
-us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_chip_eligible, violating the required output contract."
-us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an unspecified higher Pennsylvania CHIP income range instead of testing approximately $61,277 against the applicable 2026 limit. That income fails the CHIP test, and Child 1 is not Medicaid-eligible."
-us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model applied a 314% FPL ceiling and used MAGI of $60,010, omitting the child-support amount included in the trace’s approximately $61,277 household income. The applicable 2026 Pennsylvania CHIP limit is exceeded, and Medicaid supplies no alternative eligibility route."
-us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model substituted an approximately 319% FPL CHIP ceiling for the applicable 2026 Pennsylvania threshold. Although Child 1 is age-eligible, approximately $61,277 exceeds the operative income limit and the child has Medicaid category NONE."
-us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model expressly treated 138%–319% FPL as the applicable Pennsylvania CHIP income band. That is the wrong threshold rule for this engine calculation: approximately $61,277 exceeds the applicable 2026 CHIP limit, and Child 1 is not Medicaid-eligible."
-us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared the child within Pennsylvania’s CHIP income range without applying the applicable 2026 threshold. The trace’s approximately $61,277 income is over that limit, and the child cannot qualify through Medicaid because Medicaid eligibility is false."
+us,scenario_028,child1_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted a general Pennsylvania CHIP ceiling of roughly 314% FPL for the applicable PolicyEngine income limit. Under the engine's 2026 test, household income of approximately $61,277 exceeds the limit, so age 10 does not produce eligibility."
+us,scenario_028,child1_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model applied a 314% FPL upper threshold and therefore placed the child inside the CHIP income range. PolicyEngine's applicable 2026 Pennsylvania limit is below the household's approximately $61,277 income, and Medicaid category NONE supplies no alternative pathway."
+us,scenario_028,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model compared $60,010 to a generic 314% FPL ceiling instead of applying the engine's applicable CHIP income threshold and income measure. The trace uses approximately $61,277, which exceeds that threshold."
+us,scenario_028,child1_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted that household income falls within Pennsylvania's CHIP limits without performing the applicable 2026 threshold comparison. PolicyEngine counts approximately $61,277, above the relevant limit, while the child's Medicaid category is NONE."
+us,scenario_028,child1_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable child1_chip_eligible output. The required result is 0 because approximately $61,277 exceeds the applicable Pennsylvania CHIP income limit and the child is not Medicaid-eligible."
+us,scenario_028,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model treated the household as falling within Pennsylvania's higher CHIP income range. The applicable PolicyEngine comparison instead places approximately $61,277 above the 2026 income limit, with no Medicaid pathway."
+us,scenario_028,child1_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model correctly recognized that Child 1 is not Medicaid-eligible but then applied an inapplicable ceiling of about 314% FPL to the CHIP test. PolicyEngine's applicable income limit is exceeded by the household's approximately $61,277 income."
+us,scenario_028,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model used $60,010 and a generic 314% FPL ceiling to establish CHIP eligibility. The engine's test uses household income of approximately $61,277 and an applicable limit below that amount."
+us,scenario_028,child1_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model placed the child between a Medicaid cutoff and a roughly 319% FPL CHIP ceiling, treating that band as sufficient for eligibility. Under PolicyEngine's applicable 2026 Pennsylvania CHIP test, approximately $61,277 exceeds the income limit, and Medicaid category NONE provides no fallback."
+us,scenario_028,child1_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly constructed a CHIP income band of 138%–319% FPL and treated $61,277 as inside it. PolicyEngine's applicable Pennsylvania CHIP income limit for this case is below $61,277."
+us,scenario_028,child1_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model correctly excluded child support from its MAGI shortcut but then compared $60,010 with a generic 300%+ FPL ceiling. PolicyEngine's traced income is approximately $61,277 and exceeds the applicable CHIP limit; age 10 and the absence of an asset test do not cure that failed income criterion."
+us,scenario_028,child1_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that Child 1 falls within the Pennsylvania CHIP income range without applying the relevant threshold. The engine's approximately $61,277 household income exceeds the applicable limit, and the child is not Medicaid-eligible."
 us,scenario_028,child1_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,child1_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,child1_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied an approximately 215% FPL threshold as Pennsylvania's Medicaid limit for children. The engine places this child at 1.82 times FPL but in no Medicaid category, so the model used a broader child-coverage threshold instead of the applicable Medicaid pathway thresholds."
@@ -1684,17 +1773,19 @@ us,scenario_028,child1_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi
 us,scenario_028,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model provided no output or explanation for child1_wic_eligible, violating the required structured-output contract."
 us,scenario_028,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly placed a 10-year-old within WIC's eligible child age range. Children qualify categorically only while under age 5, so Child 1 is not eligible."
 us,scenario_028,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model misapplied WIC's child age limit by treating age 10 as within it. Because WIC child eligibility ends at age 5, the income test cannot make this child eligible."
-us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model treated income below approximately 314% FPL as establishing Pennsylvania CHIP eligibility. Child 2 passes the age test and is not Medicaid-eligible, but fails PolicyEngine’s separate CHIP income criterion."
-us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,The model compared its approximate household-income-to-FPL ratio only with a headline 314% FPL ceiling. PolicyEngine’s mandatory CHIP income test fails for Child 2 even though the child is age 10 and not Medicaid-eligible.
-us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,The model reduced CHIP eligibility to being under age 19 and below a general Pennsylvania CHIP income ceiling. Child 2 passes the age condition but fails the CHIP income criterion applied by PolicyEngine.
-us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used $60,010 and a 314% FPL upper threshold as a complete CHIP test. That shortcut misses PolicyEngine’s separate CHIP income requirement, which Child 2 fails."
-us,scenario_028,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted that household income was within Pennsylvania’s CHIP limits without applying PolicyEngine’s actual CHIP income test. Age 10 satisfies the age condition, but the income condition fails."
-us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no value or explanation for child2_chip_eligible. It therefore failed the required output contract rather than completing the eligibility determination.
-us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model relied on Pennsylvania’s broadly described higher CHIP income range and concluded that age 10 plus income within that range was sufficient. Child 2 instead fails PolicyEngine’s mandatory CHIP income criterion.
-us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model treated MAGI of $60,010 below 314% FPL as dispositive. PolicyEngine’s CHIP income test still fails for Child 2; the child’s age and Medicaid ineligibility do not independently confer CHIP eligibility."
-us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model placed the household between a presumed child-Medicaid cutoff and a roughly 319% FPL CHIP ceiling and treated that interval as automatic CHIP eligibility. Although Child 2 is not Medicaid-eligible and passes the age test, the separate PolicyEngine CHIP income condition fails."
-us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model constructed a generic 138%–319% FPL CHIP income band and placed its $61,277 income estimate inside it. That band is not the complete PolicyEngine test, under which Child 2 fails the CHIP income criterion."
-us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,The model asserted that Child 2 was within Pennsylvania’s CHIP income range without calculating the income condition used by PolicyEngine. The child passes the age test but fails that required income test.
+us,scenario_028,child2_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model treated Pennsylvania's approximate 314% FPL upper limit as the complete CHIP income test. PolicyEngine's child-specific CHIP income criterion fails for child2, so being age 10 and below that broad ceiling does not produce eligibility."
+us,scenario_028,child2_chip_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model converted roughly $60,000 of household income to 187% FPL and assumed that falling below 314% FPL conclusively establishes CHIP eligibility. It omitted the engine's child-specific CHIP income test, which child2 fails."
+us,scenario_028,child2_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,The model reduced eligibility to being under 19 and at approximately 190% FPL. Child2 satisfies the age condition but fails PolicyEngine's Pennsylvania CHIP income criterion.
+us,scenario_028,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied only a 314% FPL ceiling to $60,010 and declared eligibility. PolicyEngine separately evaluates child2's CHIP income condition, and that condition fails."
+us,scenario_028,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,The model asserted that household income was within Pennsylvania's CHIP limits without performing the engine's child-specific income-threshold test. That income condition fails even though child2 meets the age condition.
+us,scenario_028,child2_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no answer for child2_chip_eligible. It therefore failed the required output contract rather than completing the CHIP eligibility determination.
+us,scenario_028,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model relied on Pennsylvania's broadly described higher CHIP income range as sufficient for eligibility. It omitted the specific PolicyEngine income test that child2 fails.
+us,scenario_028,child2_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model correctly recognized Medicaid ineligibility as a CHIP prerequisite but incorrectly treated income below approximately 314% FPL as satisfying CHIP's independent income condition. PolicyEngine's child-specific CHIP income test fails, so the prerequisite alone cannot yield eligibility."
+us,scenario_028,child2_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model compared $60,010 only with a 314% FPL upper limit and treated that comparison as the full income rule. The engine's child-specific Pennsylvania CHIP income criterion fails for child2."
+us,scenario_028,child2_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,The model inferred CHIP eligibility from income being above a presumed Medicaid cutoff and below a presumed 319% FPL CHIP ceiling. PolicyEngine does not obtain a passing CHIP income result from that two-bound shortcut; child2's independent CHIP income condition fails.
+us,scenario_028,child2_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model invented a broad 138%-319% FPL CHIP band and placed $61,277 within it. That band comparison does not reproduce PolicyEngine's child-specific CHIP income test, which child2 fails."
+us,scenario_028,child2_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model treated approximately 185% FPL being below Pennsylvania's upper CHIP threshold as dispositive. Child2 meets the age condition, but PolicyEngine's separate CHIP income criterion fails."
+us,scenario_028,child2_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that child2 was within Pennsylvania's CHIP income range without applying the engine's actual child-specific income test. That test fails, producing no CHIP eligibility."
 us,scenario_028,child2_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,child2_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,child2_medicaid_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an incorrect 215% FPL Medicaid threshold for Pennsylvania children. At the engine’s 1.82-times-FPL income level, Child 2 exceeds every applicable Medicaid threshold and receives no Medicaid category."
@@ -1707,16 +1798,18 @@ us,scenario_028,child2_medicare_eligible,glm-5.2,parse_contract_failure,missing_
 us,scenario_028,child2_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable child2_wic_eligible output, violating the required submission contract."
 us,scenario_028,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 10 as within WIC's child age category; WIC child eligibility ends at age five. It also failed to apply the 185%-of-poverty income ceiling, which this household's approximately $61,277 income exceeds."
 us,scenario_028,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that a 10-year-old is within WIC's child age limit, which ends at age five. It also incorrectly classified the household as income-eligible despite approximately $61,277 exceeding the applicable 185%-of-poverty ceiling."
-us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted a 314% FPL upper limit of about $100,951 for the applicable PolicyEngine CHIP income threshold. Under the applicable test, approximately $61,277 exceeds the limit, so age 10 does not establish eligibility."
-us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model treated an estimated income near 190% FPL as automatically within Pennsylvania's applicable CHIP range. PolicyEngine's 2026 CHIP income test rejects the household's approximately $61,277, and Child 3 has no Medicaid eligibility pathway."
-us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a 314% FPL CHIP ceiling and understated household income as $60,010 by omitting the $1,267 of child support included in the trace's approximately $61,277. The applicable PolicyEngine income test rejects $61,277."
-us,scenario_028,child3_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted that household income was within Pennsylvania's CHIP limit without applying the applicable 2026 threshold. The trace's approximately $61,277 exceeds that threshold; only the age condition is satisfied."
-us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child3_chip_eligible, violating the required output contract."
-us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model placed the household within a higher Pennsylvania CHIP income range instead of applying the applicable PolicyEngine threshold. Approximately $61,277 exceeds that threshold, and Child 3 is not Medicaid-eligible."
-us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used an approximate 319% FPL upper limit and concluded that income near 187% FPL qualified. The applicable PolicyEngine CHIP income test rejects approximately $61,277, so being under age 19 and above the Medicaid limit does not produce eligibility."
-us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model imposed a self-calculated CHIP band of 138%–319% FPL, approximately $43,000–$99,500, rather than the applicable PolicyEngine threshold. Under the applicable test, the stated $61,277 exceeds the CHIP income limit."
-us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly treated Pennsylvania CHIP as having no income limit and therefore made age the controlling criterion. PolicyEngine applies an income criterion that the household's approximately $61,277 fails; Child 3's age 10 only satisfies the age condition."
-us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model declared the child within Pennsylvania's CHIP income range without applying the applicable threshold. The household's approximately $61,277 exceeds that limit, and Child 3 has no Medicaid eligibility pathway."
+us,scenario_028,child3_chip_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted a general 314% FPL CHIP ceiling for PolicyEngine’s applicable Pennsylvania eligibility threshold and also understated household income as $60,010 by omitting child support. Under the applicable test, approximately $61,277 exceeds the CHIP income limit, and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model treated an asserted income level of roughly 190% FPL as automatically within Pennsylvania CHIP limits instead of applying PolicyEngine’s applicable threshold. Household income of approximately $61,277 exceeds that threshold, and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model applied a general 314% FPL ceiling rather than PolicyEngine’s applicable Pennsylvania CHIP threshold and used $60,010 instead of the trace income of approximately $61,277. The traced income exceeds the applicable limit, and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted that household income fell within Pennsylvania CHIP limits without applying the applicable PolicyEngine threshold. Approximately $61,277 exceeds that limit, while child3’s age is qualifying but Medicaid eligibility is false."
+us,scenario_028,child3_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child3_chip_eligible. The required output was 0 because approximately $61,277 exceeds the applicable CHIP income threshold and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on Pennsylvania’s broader CHIP income range without applying PolicyEngine’s applicable income cutoff. The household’s approximately $61,277 exceeds that cutoff, and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model correctly recognized that child3 is not Medicaid-eligible but incorrectly compared household MAGI with a general 314% FPL ceiling. PolicyEngine’s applicable CHIP threshold is below the traced income of approximately $61,277, so the income criterion fails."
+us,scenario_028,child3_chip_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model treated the interval between its asserted Medicaid limit and a 319% FPL upper ceiling as the applicable CHIP eligibility range. PolicyEngine’s applicable Pennsylvania threshold is exceeded by approximately $61,277, and child3 is classified as not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model compared $61,277 with a generic 138%–319% FPL range instead of the applicable PolicyEngine Pennsylvania CHIP threshold. The traced income exceeds the applicable limit, and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model used a broad Pennsylvania CHIP income band and understated counted household income as $60,010 by excluding the $1,267 of child support reflected in the trace. The traced total of approximately $61,277 exceeds the applicable CHIP limit, and child3 is not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly treated Pennsylvania CHIP as having no income limit and therefore made age under 19 sufficient. PolicyEngine applies an income criterion that approximately $61,277 fails, and child3 is also not Medicaid-eligible."
+us,scenario_028,child3_chip_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that child3 fell within the Pennsylvania CHIP income range without performing the applicable threshold comparison. Household income of approximately $61,277 exceeds the applicable limit, and child3 is not Medicaid-eligible."
 us,scenario_028,child3_early_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,child3_head_start_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,child3_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated a higher CHIP-funded coverage threshold of 215% FPL as a Medicaid threshold. Child3's 1.82-times-FPL MAGI exceeds the applicable Medicaid limits, and CHIP eligibility cannot be substituted for Medicaid eligibility."
@@ -1746,89 +1839,94 @@ us,scenario_028,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,
 us,scenario_028,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA provisions expire, reverting to a small standard deduction, 2017-style brackets, and a $1,000-per-child CTC. Under the applicable 2026 parameters, taxable income is $35,860 and the $6,600 CTC fully offsets the $3,949.20 tax."
 us,scenario_028,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response failed the required output contract."
 us,scenario_028,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model subtracted a fixed $5,100 maximum ACTC from the $6,000 total CTC first and allowed only the remaining $900 to offset tax. The nonrefundable amount is instead the CTC needed to eliminate the $3,949.20 liability, while the unused $2,650.80 becomes refundable."
-us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model submitted the full $5,100 per-child ACTC cap even though its own reasoning calculated that tax liability left only a smaller unused CTC amount. It also understated the residual EITC instead of applying the 2026 phaseout to obtain $624.13."
-us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $1,267 of nontaxable child support in AGI and then invented a $3,850 ACTC limit that does not follow from its stated 15% calculation. The correct computation leaves $2,650.80 of CTC refundable and retains a $624.13 EITC."
-us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used a $6,000 total CTC instead of the 2026 $6,600 amount and treated the EITC as fully phased out. Applying the correct CTC amount and tax offset yields $2,650.80 refundable CTC, while the EITC phaseout leaves $624.13."
-us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The submitted $4,500 contradicts the model's own estimate of about $1,200 of refundable CTC and zero EITC. Independently, it used a $6,000 CTC and erased the residual $624.13 EITC instead of deriving $2,650.80 plus $624.13."
-us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model treated the $5,100 ACTC ceiling as an automatic refundable award and failed to limit the credit to the CTC remaining after the nonrefundable portion offsets tax. That remainder is $2,650.80, and the EITC phaseout still produces $624.13."
-us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $2,000-per-child CTC and approximate deduction and bracket parameters, producing the wrong tax offset and unused CTC. It also applied an obsolete EITC cutoff and eliminated the $624.13 residual credit."
-us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model used the wrong total CTC and then replaced its own ACTC formula with an unsupported $2,000 estimate. It also treated the EITC as fully phased out, omitting the $624.13 remaining under the 2026 phaseout."
-us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model incorrectly concluded that tax liability consumed the entire CTC and that the EITC was fully phased out. The tax offset leaves $2,650.80 refundable CTC, and the three-child EITC phaseout leaves $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated the pre-credit tax offset so severely that it left only $66 of refundable CTC rather than $2,650.80. It also understated the phaseout calculation for EITC, which yields $624.13 rather than $524."
-us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The zero answer treats both refundable-credit components as unavailable at this income. The correct computation leaves $2,650.80 of unused CTC refundable and $624.13 of EITC after phaseout."
-us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model incorrectly treated the entire CTC as used against tax liability, eliminating the $2,650.80 refundable remainder. Its EITC phaseout calculation also understated the correct $624.13 amount."
-us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used only $3,000 of total CTC for three children instead of the applicable $6,600 and therefore eliminated the refundable remainder. It also incorrectly treated the three-child EITC as fully phased out rather than retaining $624.13."
-us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model overstated the amount of CTC used nonrefundably as $4,089.20. Only $3,949.20 is used against tax, leaving $2,650.80 refundable, and the model also omitted the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model incorrectly treated the CTC as fully absorbed by tax and the EITC as fully phased out. The computation instead yields $2,650.80 refundable CTC and a $624.13 residual EITC."
-us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required output contract."
-us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the full child credit as refundable at approximately $2,000 per child, ignoring that CTC first offsets income-tax liability and that refundability is limited to the unused portion. The refundable CTC is $2,650.80, and the model also omitted the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed the income level precluded all refundable credits without applying either component's calculation. The CTC leaves $2,650.80 refundable after the tax offset, and the EITC phaseout leaves $624.13."
-us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model understated federal tax used to absorb the nonrefundable CTC as $2,184.20 instead of $3,949.20, thereby overstating refundable CTC. It also eliminated the $624.13 EITC that remains after the 2026 phaseout."
-us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly identified the $2,650.80 refundable CTC but incorrectly declared the EITC unavailable. Applying the three-child 2026 phaseout adds $624.13."
-us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model awarded the full $5,100 ACTC cap solely because earnings satisfied the phase-in formula, ignoring the separate limit based on unused CTC after tax liability. That limit produces $2,650.80, and the model also omitted the $624.13 EITC."
-us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly ruled out all refundable credits based on income. The household retains $624.13 of EITC after phaseout and has $2,650.80 of CTC left refundable after its nonrefundable use."
-us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model's assertion that only about $40 of CTC remains implies a grossly overstated nonrefundable tax offset; the actual refundable CTC remainder is $2,650.80. Its approximate EITC also exceeds the correct phaseout result of $624.13."
-us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC, causing it to treat the entire child credit as nonrefundable, instead of applying the 2026 $6,600 total and deriving $2,650.80 refundable. Its projected EITC parameters also overstated the residual credit, which is $624.13."
-us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model essentially derived the refundable CTC but understated the EITC by using an incorrect phaseout result of about $340. The correct three-child EITC at the applicable income is $624.13, which combines with $2,650.80 of refundable CTC."
-us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required output contract."
-us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly calculated the $624.13 EITC but understated the amount of CTC used nonrefundably as $3,838.10. The correct tax offset is $3,949.20, leaving $2,650.80 refundable rather than $2,761.90."
-us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the earned-income phase-in and $5,100 cap as the only ACTC limits, omitting the limit to unused CTC after nonrefundable application. It also incorrectly set EITC to zero; the two correct components are $2,650.80 and $624.13."
-us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model repeatedly substituted projected parameters for the applicable 2026 CTC, deduction, tax-bracket, and EITC values. It used a $6,000 CTC and overstated the tax offset, while its final EITC estimate of $1,042 exceeded the correct $624.13 phaseout result."
-us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model treated the refundable CTC as a standalone earned-income calculation and produced an unsupported $3,200, without subtracting the CTC used nonrefundably against tax. The correct refundable CTC is $2,650.80, and the omitted EITC is $624.13."
+us,scenario_028,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model's own intermediate work produced residual EITC and unused CTC, but it discarded that computation and submitted the $5,100 per-child ACTC ceiling. The ceiling is not the refundable amount: refundable CTC is limited to $2,650.80 of unused CTC, and the residual EITC is $624.13."
+us,scenario_028,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly included nontaxable child support in MAGI, eliminated the residual EITC, and invented a $3,850 ACTC limitation that does not follow from its 15% calculation. The correct components are $624.13 of EITC and $2,650.80 of refundable CTC."
+us,scenario_028,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used outdated projected CTC and deduction parameters, treating total CTC as $6,000 instead of $6,600 and calculating the wrong tentative tax. It also declared the EITC fully phased out instead of retaining $624.13, while the correct unused refundable CTC is $2,650.80."
+us,scenario_028,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly described an unused-credit limitation but then submitted $4,500, which is unrelated to its own estimated $1,200 ACTC. It also used a $6,000 total CTC and eliminated the $624.13 residual EITC; the refundable CTC is $2,650.80."
+us,scenario_028,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model treated the $5,100 ACTC per-child cap as an automatic refundable payment and omitted the unused-total-CTC limitation. Only $2,650.80 remains after nonrefundable CTC is used, and the EITC phaseout leaves $624.13 rather than zero."
+us,scenario_028,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used outdated projected 2026 standard-deduction, bracket, and $2,000-per-child CTC amounts, producing only $1,830 of unused CTC. It also applied an obsolete EITC endpoint and eliminated the $624.13 residual credit; refundable CTC is $2,650.80."
+us,scenario_028,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model used the wrong total CTC and tax calculation, then replaced its own $5,100 ceiling calculation with an unexplained $2,000 estimate. It also treated EITC as fully phased out, whereas the correct components are $2,650.80 of refundable CTC and $624.13 of EITC."
+us,scenario_028,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model incorrectly asserted that tax liability exceeds the CTC, eliminating refundable CTC, and also treated the EITC as fully phased out. The CTC exceeds the tax absorbed nonrefundably by $2,650.80, and the EITC phaseout leaves $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model calculated only $66 of unused CTC, reflecting an overstated tentative tax or understated total CTC. The unused refundable CTC is $2,650.80, and its $524 EITC estimate also understates the phaseout result of $624.13."
+us,scenario_028,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model eliminated both refundable-credit components without applying their formulas. The household retains $624.13 of EITC in the three-child phaseout range and $2,650.80 of CTC unused after the nonrefundable offset."
+us,scenario_028,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model understated the residual EITC as $574 and incorrectly claimed the entire CTC was consumed by tax liability. The phaseout yields $624.13 of EITC, and $2,650.80 of the $6,600 CTC remains refundable."
+us,scenario_028,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC, half of the old $6,000 amount and far below the applicable $6,600 credit, so it wrongly found no unused CTC. It also applied the wrong EITC cutoff; $624.13 remains after phaseout."
+us,scenario_028,federal_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $4,089.20 as the nonrefundable CTC absorbed by tax rather than $3,949.20, understating refundable CTC by $740. It also omitted the $624.13 EITC entirely."
+us,scenario_028,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model incorrectly treated the EITC as fully phased out and the CTC as fully absorbed nonrefundably. The correct calculation retains $624.13 of EITC and leaves $2,650.80 of CTC refundable."
+us,scenario_028,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for federal_refundable_credits.
+us,scenario_028,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the entire child credit as refundable at an inapplicable $2,000 per child and ignored both the nonrefundable tax offset and the ACTC limitations. Refundable CTC is $2,650.80, and the separate EITC adds $624.13."
+us,scenario_028,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed the income level precluded every refundable credit without applying the three-child EITC phaseout or unused-CTC calculation. Those computations yield $624.13 of EITC and $2,650.80 of refundable CTC."
+us,scenario_028,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model understated the tax absorbed by nonrefundable CTC as $2,184.20 instead of $3,949.20, thereby overstating refundable CTC. It also eliminated the $624.13 residual EITC."
+us,scenario_028,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly obtained the $2,650.80 refundable CTC but incorrectly declared EITC unavailable. The three-child 2026 phaseout leaves $624.13, which must be added to the CTC component."
+us,scenario_028,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model treated the $1,700-per-child ACTC cap as the amount paid without limiting it to unused total CTC after the nonrefundable offset. That limitation produces $2,650.80, and the model also omitted the $624.13 EITC."
+us,scenario_028,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly concluded that income disqualified the household from all refundable credits. The EITC phaseout retains $624.13, and $2,650.80 of CTC remains after applying its nonrefundable portion."
+us,scenario_028,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model reduced unused CTC to about $40 by using the wrong total CTC or overstating tax absorbed nonrefundably; the correct unused amount is $2,650.80. Its projected EITC parameters also yield $660 rather than the traced $624.13."
+us,scenario_028,federal_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model calculated only $48 of unused CTC instead of $2,650.80 because it used the wrong CTC-versus-tax-liability computation. It also used projected EITC parameters that produced $636 rather than the applicable $624.13."
+us,scenario_028,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $3,000 total CTC and therefore wrongly concluded it was entirely nonrefundable; the applicable total CTC is $6,600, leaving $2,650.80 refundable. Its projected EITC maximum and phaseout threshold also overstate the residual credit, which is $624.13."
+us,scenario_028,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model essentially identified the correct refundable CTC but used the wrong 2026 EITC phaseout parameters, producing about $340 instead of $624.13. Adding the traced components yields $3,274.93."
+us,scenario_028,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for federal_refundable_credits.
+us,scenario_028,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly calculated the $624.13 EITC but understated tax absorbed by the nonrefundable CTC as $3,838.10. The correct $3,949.20 nonrefundable use leaves $2,650.80 refundable, not $2,761.90."
+us,scenario_028,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the $5,100 ACTC cap as payable without limiting the credit to the unused portion of total CTC after tax liability. That unused portion is $2,650.80, and the EITC phaseout leaves $624.13 rather than zero."
+us,scenario_028,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model correctly calculated refundable CTC as $2,650.80 but used the wrong EITC phaseout threshold, producing $273.32. The applicable phaseout calculation yields $624.13 of EITC."
+us,scenario_028,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model repeatedly substituted projected parameters, used a $6,000 total CTC instead of $6,600, and overstated tentative tax, reducing ACTC to $1,918.80. The applicable calculations yield $2,650.80 of refundable CTC and $624.13 of EITC."
+us,scenario_028,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model gave an unexplained $3,200 ACTC amount and mischaracterized the earned-income formula as the binding limit. Refundable CTC is instead limited by the $2,650.80 of unused total CTC, and the household also receives $624.13 of EITC."
 us,scenario_028,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived $3,720 of Social Security tax, $870 of Medicare tax, and $42 of Pennsylvania employee unemployment contribution, totaling $4,632, but submitted $5,730. Its final value contradicts its own complete arithmetic."
-us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly set Pennsylvania's mandatory employee payroll contribution to zero. The 0.07% employee unemployment-compensation contribution adds $42 on $60,000 of wages to the $4,590 federal FICA total."
-us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no mandatory employee payroll tax counted here. It omitted the 0.07% employee unemployment-compensation contribution, equal to $42 on $60,000."
+us,scenario_028,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived $3,720 of Social Security tax, $870 of Medicare tax, and $42 of Pennsylvania employee unemployment compensation, totaling $4,632, but submitted $5,730 instead. Its numeric output contradicts its completed computation."
+us,scenario_028,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated Pennsylvania's employee payroll contribution as zero. The mandatory 0.07% employee unemployment-compensation contribution adds $42 to the $4,590 federal FICA total."
+us,scenario_028,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no mandatory employee payroll tax counted here. It omitted the 0.07% employee unemployment-compensation contribution, equal to $42 on $60,000 of wages."
 us,scenario_028,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,The model stopped after calculating federal Social Security and Medicare taxes. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution of $42.
-us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model conflated the absence of a general state payroll tax with the absence of all mandatory employee contributions. Pennsylvania's employee unemployment-compensation contribution is 0.07% of wages and adds $42.
-us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,The model incorrectly asserted that Pennsylvania has no employee UI contribution. The mandatory 0.07% employee unemployment-compensation contribution adds $42 to federal FICA.
-us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,The model calculated only federal Social Security and Medicare taxes. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution of $42.
-us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model limited payroll tax to federal Social Security and Medicare. It failed to add Pennsylvania's mandatory employee unemployment-compensation contribution of $42.
-us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output. It therefore failed the required structured-output contract.
-us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model incorrectly stated that no mandatory Pennsylvania employee payroll tax applies. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $42 on the stated wages.
-us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model misadded its stated federal components: $3,720 of Social Security plus $870 of Medicare equals $4,590, not $4,600. It also omitted Pennsylvania's $42 employee unemployment-compensation contribution, so the complete total is $4,632."
-us,scenario_028,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model incorrectly excluded all mandatory Pennsylvania employee payroll contributions. The state's 0.07% employee unemployment-compensation contribution adds $42 to the $4,590 FICA amount."
-us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model recognized Pennsylvania's employee unemployment contribution but calculated it as $7. The applicable 0.07% rate on $60,000 produces $42, yielding total payroll tax of $4,632."
-us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model treated federal FICA as the entire employee payroll-tax liability. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $42.
-us,scenario_028,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model applied only the 7.65% federal FICA rate. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution, which adds $42."
-us,scenario_028,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model explicitly but incorrectly excluded Pennsylvania employee payroll tax. The mandatory employee unemployment-compensation contribution is 0.07% of $60,000, or $42."
-us,scenario_028,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model stopped after federal Social Security and Medicare taxes. It omitted the $42 Pennsylvania employee unemployment-compensation contribution.
-us,scenario_028,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model calculated federal FICA but did not include the mandatory Pennsylvania employee unemployment-compensation contribution. That state contribution equals $42 on $60,000 of wages."
-us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax output. It therefore failed the required structured-output contract.
-us,scenario_028,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model incorrectly stated that PolicyEngine applies no Pennsylvania employee state payroll tax. PolicyEngine includes the 0.07% employee unemployment-compensation contribution, equal to $42."
-us,scenario_028,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model incorrectly treated Pennsylvania's mandatory employee state payroll tax as zero. The 0.07% employee unemployment-compensation contribution adds $42 to the federal payroll taxes.
-us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly claimed Pennsylvania has no employee unemployment deduction. The state's mandatory employee unemployment-compensation contribution is 0.07% of wages, producing $42 here."
-us,scenario_028,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model treated the 7.65% federal Social Security and Medicare amount as the full payroll tax. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of $42.
-us,scenario_028,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model counted child support in its school-meal income calculation and then applied an income limit that put $61,267 above the ceiling. The applicable calculation produces an income-to-guideline ratio of 1.82, which is below 1.85 and assigns the REDUCED tier."
-us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model counted child support and interest and used an approximate $59,478 ceiling instead of the applicable 2026 income-to-guideline calculation. That calculation yields 1.82, below the 1.85 reduced-price limit."
-us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an understated approximate 185% guideline of $57,720. Under the applicable guideline, the household's ratio is 1.82, so it falls within the reduced-price band."
-us,scenario_028,reduced_price_school_meals_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model assigned an income ratio of approximately 1.90 instead of the engine-derived 1.82. Because 1.82 is below 1.85, the household is in the REDUCED tier."
-us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model added child support and interest to wages and paired that total with an understated approximate guideline. The applicable school-meal calculation yields a ratio of 1.82, below the 1.85 ceiling."
-us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly called one adult plus three children a household of five and then contradicted its own stated threshold: $61,277 is below its approximate $66,000 ceiling. The applicable household calculation yields a 1.82 ratio and REDUCED eligibility."
-us,scenario_028,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an approximate $61,200 cutoff and treated its gross-income total as slightly above the limit. The applicable calculation instead produces a 1.82 ratio, below the 1.85 cutoff."
-us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted that income exceeded the limit without calculating the applicable income-to-guideline ratio. That ratio is 1.82, below the 1.85 reduced-price threshold."
-us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model treated $61,277, including child support, as above the reduced-price limit. The applicable school-meal calculation yields a ratio of 1.82, which satisfies the 1.85 ceiling."
-us,scenario_028,reduced_price_school_meals_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an understated approximate 185% cutoff of $57,720. The applicable guideline calculation places the household at 1.82 of poverty, within the reduced-price band."
-us,scenario_028,reduced_price_school_meals_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model asserted that the household exceeded 185% of poverty without deriving the ratio. The applicable ratio is 1.82, so the household qualifies for the REDUCED tier."
-us,scenario_028,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no answer for the requested variable. It therefore failed the required output contract before any eligibility computation could be evaluated.
-us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model declared income above the reduced-price threshold without applying the applicable guideline calculation. That calculation yields 1.82, below the 1.85 ceiling."
-us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of an explicit eligibility indicator as grounds for denial instead of deriving income-based eligibility from the supplied household facts. The income test yields a 1.82 ratio and assigns the REDUCED tier.
-us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model included child support and concluded that the resulting total exceeded the four-person limit. The applicable school-meal calculation produces a ratio of 1.82, below the 1.85 limit."
-us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model added wages, interest, and child support and compared the result with an estimated poverty threshold. The applicable calculation yields an income-to-guideline ratio of 1.82, not a value above 1.85."
-us,scenario_028,reduced_price_school_meals_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model asserted that income was above the threshold without computing the governing ratio. The ratio is 1.82, which is below the 1.85 reduced-price ceiling."
-us,scenario_028,reduced_price_school_meals_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model included child support in its $61,277 comparison and placed the household above 185% of poverty. The applicable school-meal calculation instead yields a 1.82 ratio and reduced-price eligibility."
-us,scenario_028,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model calculated an approximate 1.86 poverty ratio from $61,277. The applicable calculation yields 1.82, below the 1.85 reduced-price threshold."
-us,scenario_028,reduced_price_school_meals_eligible,inkling,llm_error,thresholds_rates,False,"The model used an approximate $59,500 ceiling and treated $61,277 as controlling school-meal income. The applicable calculation yields a ratio of 1.82, which qualifies for reduced-price meals."
-us,scenario_028,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model treated wages, child support, and interest totaling $61,277 as countable and compared that total with a projected guideline. The applicable school-meal computation produces a 1.82 ratio, below the 1.85 limit."
-us,scenario_028,reduced_price_school_meals_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used an understated approximate four-person poverty guideline and consequently calculated ratios of 1.87 to 1.91. The applicable ratio is 1.82, placing the household below the reduced-price ceiling."
-us,scenario_028,reduced_price_school_meals_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model used an approximate $57,700 reduced-price limit and compared it with $61,277. The applicable guideline calculation yields a 1.82 ratio, within the reduced-price band."
-us,scenario_028,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly described one adult and three children as a household of five and used an approximate $59,000–$61,000 ceiling. The applicable household calculation yields a 1.82 ratio and REDUCED eligibility."
-us,scenario_028,reduced_price_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model asserted that income exceeded the reduced-price limit without calculating the applicable ratio. The ratio is 1.82, below the 1.85 cutoff, so positive reduced-price support results."
+us,scenario_028,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model incorrectly equated Pennsylvania's lack of a separate disability tax with having no employee-side state payroll tax. Pennsylvania's employee unemployment-compensation contribution is 0.07% of wages and adds $42.
+us,scenario_028,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no employee unemployment contribution. The mandatory 0.07% employee unemployment-compensation contribution adds $42 on $60,000 of wages."
+us,scenario_028,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,The model calculated only federal Social Security and Medicare taxes. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $42.
+us,scenario_028,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model limited payroll tax to Social Security and Medicare. Pennsylvania's mandatory employee unemployment-compensation contribution adds $42 to that $4,590 federal total."
+us,scenario_028,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, violating the required structured-output contract."
+us,scenario_028,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly stated that no mandatory state payroll tax applies in Pennsylvania. It omitted the 0.07% employee unemployment-compensation contribution, which equals $42."
+us,scenario_028,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model misadded $3,720 of Social Security tax and $870 of Medicare tax as $4,600 instead of $4,590. It also omitted Pennsylvania's $42 employee unemployment-compensation contribution, so the complete total is $4,632."
+us,scenario_028,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,The model incorrectly excluded all mandatory Pennsylvania employee payroll taxes. The state's 0.07% employee unemployment-compensation contribution adds $42 to federal FICA.
+us,scenario_028,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model recognized Pennsylvania employee unemployment tax but calculated it as $7. The applicable 0.07% rate on $60,000 produces $42, making total payroll tax $4,632."
+us,scenario_028,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model treated the 7.65% federal FICA calculation as the entire employee payroll tax. It omitted Pennsylvania's mandatory $42 employee unemployment-compensation contribution.
+us,scenario_028,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model stopped at federal FICA of 7.65% of wages. It omitted Pennsylvania's 0.07% employee unemployment-compensation contribution of $42.
+us,scenario_028,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no employee payroll tax. The mandatory employee unemployment-compensation contribution is 0.07% of wages, adding $42."
+us,scenario_028,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,The model incorrectly asserted that Pennsylvania has no mandatory employee state payroll tax. It omitted the $42 employee unemployment-compensation contribution.
+us,scenario_028,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model calculated federal Social Security and Medicare taxes but did not include the Pennsylvania employee contribution. The state's 0.07% unemployment-compensation tax adds $42.
+us,scenario_028,payroll_tax,inkling,llm_error,payroll_tax_base,False,The model stopped after federal Social Security and Medicare taxes. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of $42.
+us,scenario_028,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, violating the required structured-output contract."
+us,scenario_028,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,The model incorrectly stated that PolicyEngine applies no employee state payroll tax in Pennsylvania. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $42.
+us,scenario_028,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no mandatory employee state payroll tax. It omitted the 0.07% employee unemployment-compensation contribution, equal to $42."
+us,scenario_028,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,The model included only federal Social Security and Medicare taxes. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $42.
+us,scenario_028,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly claimed that Pennsylvania has no employee unemployment deduction beyond income-tax withholding. Pennsylvania imposes a 0.07% employee unemployment-compensation contribution, adding $42."
+us,scenario_028,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model treated federal Social Security and Medicare taxes as the complete payroll-tax liability. It omitted Pennsylvania's mandatory $42 employee unemployment-compensation contribution.
+us,scenario_028,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an incorrect 2026 poverty threshold and declared $61,267 above 185% FPG. The engine’s income calculation produces a 1.82 FPG ratio, which is below the 1.85 reduced-price limit."
+us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,thresholds_rates,False,"The model applied an approximate 2025–26 limit of $59,478 instead of the benchmark’s 2026 income-to-FPG calculation. The resulting ratio is 1.82, so the household falls within the reduced-price tier."
+us,scenario_028,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an incorrect approximate 185% FPG threshold of $57,720. Under the applicable 2026 parameters, the household’s school-meal FPG ratio is 1.82 and therefore does not exceed 1.85."
+us,scenario_028,reduced_price_school_meals_eligible,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated the household at roughly 190% FPG instead of computing the applicable school-meal ratio. That ratio is 182%, placing the household below the 185% reduced-price ceiling."
+us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model relied on an approximate and understated family-of-four poverty threshold, then placed $61,277 above the limit. The applicable computation yields 1.82 FPG, which satisfies the reduced-price income test."
+us,scenario_028,reduced_price_school_meals_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model both miscounted the four-person household as five people and contradicted its own stated threshold: $61,277 is below its stated $66,000 limit. The engine’s actual ratio is 1.82, below the 1.85 ceiling."
+us,scenario_028,reduced_price_school_meals_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model substituted an approximate $61,200 cutoff and treated the household as just over it. The applicable computation yields a 1.82 FPG ratio, safely below the 1.85 cutoff."
+us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted that income exceeded the reduced-price limit without applying the benchmark’s FPG calculation. That calculation produces a ratio of 1.82, which satisfies the 1.85 limit."
+us,scenario_028,reduced_price_school_meals_eligible,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly placed its $61,277 gross-income figure above 185% FPG. The school-meal computation yields an income-to-FPG ratio of 1.82 and assigns the REDUCED tier."
+us,scenario_028,reduced_price_school_meals_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an understated approximate cutoff of $57,720. With the applicable 2026 guideline, the household is at 1.82 FPG and remains below the 1.85 reduced-price threshold."
+us,scenario_028,reduced_price_school_meals_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly asserted that household income exceeded 185% FPG. The engine’s income test places it at 182% FPG, so reduced-price support is positive."
+us,scenario_028,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no answer for the requested output. It therefore failed the required output contract rather than performing the reduced-price income test.
+us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model declared income above the reduced-price threshold without computing the applicable ratio. The ratio is 1.82, below the 1.85 ceiling, so the household receives reduced-price support."
+us,scenario_028,reduced_price_school_meals_eligible,gpt-5.4-nano,llm_error,thresholds_rates,False,The model treated the absence of an explicit eligibility indicator in the prompt as evidence of ineligibility and skipped the required income test. The supplied household facts produce a 1.82 FPG ratio and REDUCED eligibility.
+us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model incorrectly treated the four-person household’s income as exceeding the applicable reduced-price limit. The correct school-meal income ratio is 1.82, which is below 1.85."
+us,scenario_028,reduced_price_school_meals_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model relied on an estimated poverty threshold that placed $61,277 slightly over the limit. The applicable computation instead yields 1.82 FPG, placing the household below the 1.85 ceiling."
+us,scenario_028,reduced_price_school_meals_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model asserted that income was above the threshold without applying the applicable FPG parameters. The resulting ratio is 1.82, so the household qualifies for the reduced-price tier."
+us,scenario_028,reduced_price_school_meals_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used the wrong 185% FPG dollar limit for a family of four. The school-meal calculation places the household at 1.82 FPG, below the reduced-price cutoff."
+us,scenario_028,reduced_price_school_meals_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model used a projected 2026 poverty threshold that incorrectly placed $61,277 above 185% FPG. The applicable ratio is 1.82; lack of categorical eligibility therefore leaves the household in the income-tested REDUCED tier, not ineligible."
+us,scenario_028,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model estimated the household at 186% FPG. The applicable school-meal computation yields 182% FPG, which is below the 185% reduced-price limit."
+us,scenario_028,reduced_price_school_meals_eligible,inkling,llm_error,thresholds_rates,False,"The model used an approximate $59,500 family-of-four limit that understated the applicable threshold. The household’s computed ratio is 1.82 FPG, so it qualifies for reduced-price meals."
+us,scenario_028,reduced_price_school_meals_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model used projected poverty levels that incorrectly put its $61,277 income total above 185% FPG. Under the applicable calculation, the household is at 1.82 FPG and qualifies for the REDUCED tier."
+us,scenario_028,reduced_price_school_meals_eligible,kimi-k3,llm_error,thresholds_rates,False,"The model used an incorrect poverty guideline and derived ratios of 187%–191%. The applicable school-meal ratio is 182%, below the 185% ceiling."
+us,scenario_028,reduced_price_school_meals_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model applied an understated approximate threshold of $57,700. The applicable guideline calculation gives a 1.82 FPG ratio, so the household is eligible for reduced-price support."
+us,scenario_028,reduced_price_school_meals_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model miscounted one adult plus three children as a five-person household and then used an approximate $59,000–$61,000 cutoff. The applicable school-meal calculation yields a 1.82 FPG ratio, below the 1.85 reduced-price limit."
+us,scenario_028,reduced_price_school_meals_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly asserted that household income was above the reduced-price limit. The computed income-to-FPG ratio is 1.82, which satisfies the 1.85 threshold."
 us,scenario_028,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_028,ssi,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -1839,36 +1937,38 @@ us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_erro
 us,scenario_028,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model applied an unspecified deduction-and-credit framework that produced $3,480, equivalent to taxing about $113,355 at Pennsylvania's 3.07% rate. The household's Pennsylvania taxable income is only $60,010, and no listed nonrefundable credit or deduction produces the submitted amount."
 us,scenario_028,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly imported a federal-style $17,800 head-of-household standard deduction into Pennsylvania's personal income tax calculation. Pennsylvania provides no such standard deduction here, so the full $60,010 wage-and-interest base is taxed at 3.07%."
 us,scenario_028,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model correctly computed 3.07% of $60,010 as $1,842.31, then improperly subtracted an invented $1.55 Pennsylvania Child and Dependent Care Tax Credit. No qualifying dependent-care expense was listed, and the traced computation retains the full $1,842.31 at this stage."
-us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Pennsylvania has no refundable state EITC and considered only Tax Forgiveness. Pennsylvania’s 2026 refundable EITC equals 10% of the $624.13 federal EITC, producing $62.41."
-us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model misclassified Pennsylvania’s 2026 EITC as nonrefundable. The refundable state credit is 10% of the household’s $624.13 federal EITC, or $62.41."
-us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,The model restricted its analysis to Tax Forgiveness and omitted Pennsylvania’s refundable 2026 EITC. The state EITC supplies $62.41 even though this household is above the Tax Forgiveness limit.
-us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model treated Tax Forgiveness as Pennsylvania’s only relevant relief mechanism and omitted the separate refundable state EITC. Applying the 10% state rate to the $624.13 federal EITC yields $62.41.
-us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The answer omitted Pennsylvania’s refundable 2026 EITC. The household receives $62.41, calculated as 10% of its $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly stated that Pennsylvania has no state EITC. Pennsylvania’s refundable 2026 EITC is 10% of the household’s $624.13 federal credit, yielding $62.41."
-us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model considered Tax Forgiveness but incorrectly denied the existence of a refundable Pennsylvania EITC. The separate 2026 state EITC equals 10% of the $624.13 federal EITC, or $62.41."
-us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model evaluated only Pennsylvania Tax Forgiveness and stopped after finding income above its threshold. It omitted the refundable state EITC, which equals 10% of the $624.13 federal EITC and produces $62.41."
-us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Pennsylvania credit applies at this income. The household qualifies for a $624.13 federal EITC, which generates a $62.41 Pennsylvania EITC at the 10% state rate."
-us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The answer omitted Pennsylvania’s refundable 2026 EITC. Applying the state’s 10% rate to the household’s $624.13 federal EITC yields $62.41.
-us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model focused on the child-care credit and treated the absence of child-care expenses as eliminating refundable state credits. Pennsylvania’s EITC does not require child-care expenses and equals 10% of the $624.13 federal EITC, producing $62.41."
-us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model failed to apply Pennsylvania’s refundable 2026 EITC. The household’s $624.13 federal EITC produces a state credit of $62.41 at 10%.
-us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly denied eligibility for Pennsylvania refundable credits. The household’s federal EITC is $624.13, making its refundable Pennsylvania EITC $62.41."
-us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model omitted the Pennsylvania refundable EITC eligibility pathway tied to the federal EITC. Ten percent of the $624.13 federal credit is $62.41.
-us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract."
-us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked Pennsylvania’s refundable 2026 EITC despite the wages and qualifying children supplied in the facts. The $624.13 federal EITC generates a $62.41 state credit.
-us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required a separate listed state-credit eligibility indicator instead of deriving the Pennsylvania EITC from federal EITC eligibility. The federal EITC is $624.13, and its 10% Pennsylvania supplement is $62.41."
-us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model tied refundable-credit eligibility to listed child-care expenses and omitted Pennsylvania’s EITC. The EITC requires no child-care expense input and equals 10% of the $624.13 federal EITC, or $62.41."
-us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The answer omitted Pennsylvania’s refundable 2026 EITC. The state credit is $62.41, calculated as 10% of the household’s $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model failed to recognize the applicable refundable Pennsylvania EITC. Applying 10% to the $624.13 federal EITC gives $62.41.
-us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly treated this income as disqualifying the household from all Pennsylvania refundable credits. The household still has a $624.13 federal EITC, creating a $62.41 Pennsylvania EITC."
-us,scenario_028,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The zero answer omitted Pennsylvania’s refundable 2026 EITC. The correct state computation is 10% of the $624.13 federal EITC, yielding $62.41."
-us,scenario_028,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable Pennsylvania personal-income-tax credit applies. The household qualifies for a $62.41 state EITC based on its $624.13 federal EITC.
-us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model expressly but incorrectly asserted that Pennsylvania has no state EITC. Pennsylvania’s refundable 2026 EITC equals 10% of the $624.13 federal EITC, producing $62.41."
-us,scenario_028,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model considered only nonrefundable Tax Forgiveness and omitted Pennsylvania’s separate refundable EITC. The state EITC equals 10% of $624.13, so the refundable credit is $62.41."
-us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly rejected Tax Forgiveness but incorrectly concluded that no other refundable credit applies. Pennsylvania’s 2026 refundable EITC supplies $62.41, equal to 10% of the household’s $624.13 federal EITC."
-us,scenario_028,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,"The model incorrectly stated that Pennsylvania has no refundable state EITC. The 2026 Pennsylvania EITC is 10% of the household’s $624.13 federal EITC, or $62.41."
-us,scenario_028,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model omitted the refundable Pennsylvania EITC applicable to this household. The $624.13 federal EITC generates a $62.41 state credit at the 10% rate.
-us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Pennsylvania does not offer a refundable EITC. Pennsylvania’s 2026 credit equals 10% of the household’s $624.13 federal EITC, yielding $62.41."
-us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The answer omitted Pennsylvania’s refundable 2026 EITC. Ten percent of the household’s $624.13 federal EITC produces $62.41 in state refundable credits.
+us,scenario_028,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly asserted that Pennsylvania has no state EITC. It omitted the refundable Pennsylvania EITC equal to 10% of the $624.13 federal EITC, yielding $62.41."
+us,scenario_028,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model misclassified Pennsylvania's 2026 EITC as nonrefundable. The refundable credit is 10% of the household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,The model treated Tax Forgiveness as the only relevant Pennsylvania credit and omitted the refundable state EITC. The state EITC supplies $62.41 regardless of the household's ineligibility for Tax Forgiveness.
+us,scenario_028,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly limited Pennsylvania refundable-credit analysis to Tax Forgiveness. It failed to apply the refundable state EITC at 10% of the $624.13 federal EITC.
+us,scenario_028,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The zero answer omits Pennsylvania's refundable EITC. Applying the 10% state percentage to the $624.13 federal EITC produces $62.41.
+us,scenario_028,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly stated that Pennsylvania has no state EITC. Pennsylvania's refundable EITC equals 10% of this household's $624.13 federal EITC, producing $62.41."
+us,scenario_028,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model incorrectly substituted the nonrefundable Tax Forgiveness program for the separate refundable Pennsylvania EITC. The latter equals 10% of the federal EITC and yields $62.41.
+us,scenario_028,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model tested only Pennsylvania Tax Forgiveness and stopped after finding income above its threshold. It omitted the refundable state EITC, which equals 10% of $624.13 and produces $62.41."
+us,scenario_028,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,The model incorrectly concluded that no refundable Pennsylvania credit applied in 2026. The household receives a $62.41 state EITC based on 10% of its $624.13 federal EITC.
+us,scenario_028,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The zero answer omits Pennsylvania's refundable EITC pathway. The required calculation is 10% of the $624.13 federal EITC, yielding $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model focused on the child-care credit and treated the absence of child-care expenses as dispositive. Pennsylvania's refundable EITC requires no child-care expense and equals 10% of the $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model omitted the refundable Pennsylvania EITC applicable to the household. Ten percent of the $624.13 federal EITC is $62.41.
+us,scenario_028,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model failed to carry the household's federal EITC into Pennsylvania's refundable EITC calculation. Applying the 10% state rate yields $62.41.
+us,scenario_028,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model incorrectly denied all refundable Pennsylvania credits. The household qualifies for $62.41 through the state EITC, calculated as 10% of its $624.13 federal EITC."
+us,scenario_028,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no output for state_refundable_credits. The required submitted amount was $62.41 from Pennsylvania's refundable EITC.
+us,scenario_028,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model failed to recognize that the household facts generate a federal EITC and therefore a Pennsylvania EITC. The state credit is 10% of $624.13, or $62.41."
+us,scenario_028,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model incorrectly required a separately listed state-credit eligibility indicator instead of computing the Pennsylvania EITC from the federal EITC. The $624.13 federal credit generates a $62.41 refundable state credit.
+us,scenario_028,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model wrongly treated the absence of child-care expenses as eliminating all refundable Pennsylvania credits. The state EITC is independent of child-care expenses and equals 10% of $624.13, producing $62.41."
+us,scenario_028,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model omitted Pennsylvania's refundable EITC. Applying the state's 10% percentage to the $624.13 federal EITC produces $62.41.
+us,scenario_028,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,The model incorrectly concluded that no Pennsylvania refundable credit applied. The household's $624.13 federal EITC generates a $62.41 Pennsylvania EITC.
+us,scenario_028,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model incorrectly treated the household's income as precluding every refundable Pennsylvania credit. The federal EITC remains $624.13 at that income, and Pennsylvania refunds 10% of it, or $62.41."
+us,scenario_028,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The zero answer omits Pennsylvania's refundable EITC. Ten percent of the household's $624.13 federal EITC yields $62.41.
+us,scenario_028,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model failed to apply Pennsylvania's refundable EITC to the household's federal EITC. The correct state computation is 10% of $624.13, yielding $62.41."
+us,scenario_028,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model incorrectly asserted that Pennsylvania has no applicable refundable individual credit. Its refundable EITC equals 10% of the $624.13 federal EITC, producing $62.41."
+us,scenario_028,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model explicitly and incorrectly denied the existence of a Pennsylvania state EITC. The refundable state EITC is 10% of the $624.13 federal credit, or $62.41."
+us,scenario_028,state_refundable_credits,inkling,llm_error,state_local_rule,False,The model treated Tax Forgiveness as the only relevant Pennsylvania relief and omitted the separate refundable EITC. That EITC equals 10% of $624.13 and supplies $62.41.
+us,scenario_028,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,The model correctly rejected Tax Forgiveness but incorrectly concluded that no other refundable credit applied. Pennsylvania's refundable EITC supplies $62.41 based on 10% of the $624.13 federal EITC.
+us,scenario_028,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model incorrectly stated that Pennsylvania has no refundable state EITC. The 2026 state EITC equals 10% of this household's $624.13 federal EITC, or $62.41."
+us,scenario_028,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model omitted the applicable Pennsylvania refundable EITC. The household's $624.13 federal EITC produces a $62.41 state credit at the 10% state percentage.
+us,scenario_028,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model used ineligibility for Tax Forgiveness to conclude that no refundable Pennsylvania credit applied. It omitted the separate state EITC, calculated as 10% of $624.13 for a $62.41 refund."
+us,scenario_028,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model explicitly and incorrectly denied Pennsylvania's refundable EITC. Applying its 10% rate to the household's $624.13 federal EITC yields $62.41.
+us,scenario_028,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model failed to identify Pennsylvania's refundable EITC. The state credit equals 10% of the $624.13 federal EITC, producing $62.41."
 us,scenario_028,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_029,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $50,000 of financial assistance as taxable federal income, inflating AGI from $312 to $50,312. It also used an erroneous $8,600 single-filer standard deduction; the correct computation applies the applicable standard deduction to the $312 of taxable interest and leaves zero taxable income."
 us,scenario_029,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $50,000 of financial assistance in federal taxable income. Only the $312 of taxable interest enters AGI, and the standard deduction reduces taxable income and federal income tax before refundable credits to zero."
@@ -1891,36 +1991,38 @@ us,scenario_029,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibi
 us,scenario_029,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model skipped the SNAP resource test and calculated an allotment solely from monthly interest income and the standard deduction. The household’s $54,985 of bank and stock assets exceeds Oklahoma’s applicable SNAP resource limit, making it ineligible before any maximum-allotment calculation."
 us,scenario_029,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $50,000 of financial assistance to the $312 of taxable interest and treated the resulting $50,312 as Oklahoma adjusted gross income. Only the $312 of taxable interest enters the tax calculation, and the applicable deduction reduces Oklahoma taxable income to zero."
 us,scenario_029,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $1,827 answer reflects treating the listed $50,000 of financial assistance as taxable Oklahoma income before applying a standard deduction. The household instead has only $312 of specified taxable income, which is fully eliminated by the applicable deduction and produces zero tax."
-us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly included $50,000 of financial assistance in Oklahoma gross income and thereby denied the Sales Tax Relief Credit; the applicable gross income is $312, producing the $40 credit."
-us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model overlooked Oklahoma's refundable Sales Tax Relief Credit, which awards this qualifying low-income taxpayer $40 despite the absence of earned income."
-us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model wrongly counted the financial-assistance input and assets against the Sales Tax Relief Credit's income test; Oklahoma gross income is $312 and assets do not disqualify the taxpayer, yielding $40."
-us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model improperly treated earned income and substantial assets or financial assistance as barriers to the Sales Tax Relief Credit; the low-income pathway uses $312 of Oklahoma gross income and grants $40.
-us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model focused on earned income and dependents and omitted the Sales Tax Relief Credit's basic low-income pathway, under which this one-person unit receives $40."
-us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model considered only earned-income and child-related credits and failed to apply Oklahoma's refundable Sales Tax Relief Credit to the taxpayer's $312 gross income, which yields $40."
-us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model overlooked the refundable Oklahoma Sales Tax Relief Credit and incorrectly made earned income, dependents, disability, or old age necessary; the basic low-income pathway produces $40."
-us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model stopped after computing a zero Oklahoma EITC and omitted the separate refundable Sales Tax Relief Credit, which contributes $40."
-us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly added $50,000 of financial assistance to Oklahoma gross income; the credit calculation uses $312 and awards the $40 Sales Tax Relief Credit."
-us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model failed to identify Oklahoma's refundable Sales Tax Relief Credit, for which the taxpayer's $312 gross income produces a $40 payment."
-us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model incorrectly treated financial assistance as income under the Sales Tax Relief Credit test; Oklahoma gross income is $312, below the applicable limit, so the credit is $40."
-us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model omitted the Oklahoma Sales Tax Relief Credit's low-income eligibility pathway, which grants this one-person unit $40."
-us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model treated age 65, disability, or dependents as mandatory, missing the separate gross-income-under-$20,000 pathway; at $312 of gross income, the age-21 taxpayer receives $40."
-us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model overlooked the refundable Oklahoma Sales Tax Relief Credit available on $312 of gross income, resulting in a missed $40."
-us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model incorrectly stated that Oklahoma offers no applicable refundable credit and omitted the $40 Sales Tax Relief Credit for this low-income taxpayer.
-us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model failed to apply Oklahoma's refundable Sales Tax Relief Credit to the one-person unit's $312 gross income, which yields $40."
-us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model relied on an absence-of-indication shortcut instead of testing Oklahoma's Sales Tax Relief Credit; the stated age, unit size, and $312 gross income establish a $40 credit."
-us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly required earned income or another earned-income-credit basis and omitted the Sales Tax Relief Credit's low-income pathway, worth $40 here."
-us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model incorrectly made earned income or a dependent necessary for every Oklahoma refundable credit; the Sales Tax Relief Credit pays this qualifying low-income taxpayer $40.
-us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model overlooked that Oklahoma's Sales Tax Relief Credit requires neither earned income nor dependents under its basic low-income pathway, which produces $40."
-us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model failed to use the listed $312 gross income, age 21, and one-person unit to apply Oklahoma's refundable Sales Tax Relief Credit of $40."
-us,scenario_029,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model incorrectly limited Oklahoma refundable credits to pathways requiring earned income, dependents, or special expenses and omitted the $40 low-income Sales Tax Relief Credit."
-us,scenario_029,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly treated qualifying children as necessary and failed to apply the Sales Tax Relief Credit's basic low-income pathway, which grants $40."
-us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model omitted Oklahoma's refundable Sales Tax Relief Credit, which contributes $40 for this taxpayer with $312 of gross income."
-us,scenario_029,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model wrongly treated income or assets as preventing an Oklahoma low-income credit; the relevant gross income is $312, assets are not part of this test, and the Sales Tax Relief Credit is $40."
-us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly required age 65, disability, dependents, or earned income and missed the Sales Tax Relief Credit's under-$20,000 gross-income pathway, which yields $40."
-us,scenario_029,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model incorrectly included $50,000 of financial assistance in gross household income; the Oklahoma calculation uses $312, qualifying the taxpayer for the $40 Sales Tax Relief Credit."
-us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely treated Oklahoma as having no state income tax and consequently omitted its refundable Sales Tax Relief Credit, which equals $40 here."
-us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model focused on earned-income and child-related credits and omitted Oklahoma's Sales Tax Relief Credit, whose low-income pathway grants this taxpayer $40."
-us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model failed to apply Oklahoma's refundable Sales Tax Relief Credit to the stated $312 gross income and qualifying one-person tax unit, missing $40."
+us,scenario_029,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $50,000 of financial assistance to the Oklahoma credit's gross-income test. The applicable gross income is $312, so the eligible one-person unit receives the $40 sales tax credit."
+us,scenario_029,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model omitted Oklahoma's refundable sales tax credit and incorrectly asserted that the state has no applicable general refundable credit. A 21-year-old filer with $312 of gross income qualifies for the $40 credit.
+us,scenario_029,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $50,000 financial assistance and treated financial assets as relevant to the sales tax credit. The credit uses $312 of gross income here and has no asset test, producing $40."
+us,scenario_029,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly treated earned income, financial assistance, and substantial assets as barriers to the Oklahoma sales tax credit. The 21-year-old head satisfies the age rule and the relevant $312 gross income yields a $40 credit."
+us,scenario_029,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model reduced Oklahoma refundable credits to earned-income and dependent-based pathways and failed to apply the sales tax credit. The single head qualifies based on age 21 and $312 of gross income, yielding $40."
+us,scenario_029,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model considered only Oklahoma's EITC and child-related credits, omitting the refundable sales tax credit. That credit does not require earned income or a child; age 21 and $312 of gross income produce $40."
+us,scenario_029,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly concluded that Oklahoma offers no applicable refundable credit after focusing on the EITC and elderly, disabled, or dependent qualifications. It omitted the sales tax credit pathway under which age 21 and $312 of gross income yield $40."
+us,scenario_029,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,The model stopped after calculating a zero Oklahoma EITC and failed to calculate the refundable sales tax credit. The latter contributes $40 based on the one-person unit's age and $312 gross income.
+us,scenario_029,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $50,000 of financial assistance in gross income for the Oklahoma sales tax credit. The relevant gross income is $312, which produces the $40 credit."
+us,scenario_029,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model failed to identify Oklahoma's refundable sales tax credit. The 21-year-old head and $312 gross income satisfy its computation for a $40 credit.
+us,scenario_029,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated financial assistance as income under the Oklahoma sales tax credit test. With applicable gross income of $312, the eligible one-person unit receives $40."
+us,scenario_029,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model omitted the applicable Oklahoma sales tax credit. Age 21 and gross income of $312 produce a refundable credit of $40.
+us,scenario_029,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly required the head to be at least 65, disabled, or have dependents. Under the applicable age rule, the 21-year-old head is eligible, and the one-person unit receives $40 at $312 gross income."
+us,scenario_029,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,The model failed to apply Oklahoma's refundable sales tax credit. The household's $312 gross income and the head's qualifying age generate $40.
+us,scenario_029,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model incorrectly stated that Oklahoma has no refundable credit for this household. Its sales tax credit supplies $40 because the head is 21 and gross income is $312.
+us,scenario_029,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model overlooked the facts supporting Oklahoma's refundable sales tax credit. The qualifying age of 21 and $312 gross income yield $40.
+us,scenario_029,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated eligibility as requiring an additional expressly listed characteristic and failed to apply the age-and-income sales tax credit rules. The supplied age 21 and $312 gross income establish a $40 credit.
+us,scenario_029,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,The model incorrectly limited refundable-credit eligibility to earned-income or other unspecified bases. Oklahoma's sales tax credit requires neither earned income nor dependents here; age 21 and $312 gross income yield $40.
+us,scenario_029,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly required earned income or a dependent for every Oklahoma refundable credit. The sales tax credit applies to this 21-year-old one-person unit with $312 gross income and equals $40.
+us,scenario_029,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly treated earned income and dependents as necessary for an Oklahoma refundable credit. It omitted the sales tax credit, which equals $40 based on age 21 and $312 gross income."
+us,scenario_029,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model failed to recognize that the listed age and taxable interest establish sales tax credit eligibility. A 21-year-old head with $312 gross income receives $40.
+us,scenario_029,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly required earned income, dependents, or special expenses for all Oklahoma refundable credits. The sales tax credit instead yields $40 from the head's qualifying age and $312 gross income."
+us,scenario_029,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated qualifying children as necessary for an Oklahoma refundable credit. The sales tax credit applies to this one-person unit based on age 21 and $312 gross income, producing $40."
+us,scenario_029,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly added financial assistance to the sales tax credit's income measure and treated investment assets as disqualifying. The applicable gross income is $312, there is no asset test in this computation, and the credit is $40."
+us,scenario_029,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model omitted Oklahoma's refundable sales tax credit. The household qualifies through the head's age of 21 and gross income of $312, generating $40."
+us,scenario_029,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model incorrectly treated dependents, earned income, and assets as controlling all Oklahoma low-income refundable credits. The sales tax credit has no asset or earned-income requirement here and equals $40 based on age 21 and $312 gross income."
+us,scenario_029,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model incorrectly required age 65, disability, dependents, earned income, or positive Oklahoma tax liability for the refundable sales tax credit. The 21-year-old head satisfies the applicable age condition, and $312 gross income produces $40."
+us,scenario_029,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $50,000 of financial assistance in gross household income for the sales tax credit. The applicable amount is $312, so the qualifying 21-year-old receives $40."
+us,scenario_029,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model falsely stated that Oklahoma has no state income tax and therefore skipped its refundable credits. Oklahoma's sales tax credit contributes $40 for this household.
+us,scenario_029,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,The model incorrectly grouped the sales tax credit with credits requiring earned income or other unlisted qualifications. The sales tax credit applies based on age 21 and $312 gross income and equals $40.
+us,scenario_029,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model considered only earned-income and child-related Oklahoma credits and omitted the sales tax credit. That credit requires neither earned income nor a child here and produces $40 from the qualifying age and $312 gross income.
+us,scenario_029,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,The model failed to identify Oklahoma's refundable sales tax credit from the supplied age and income facts. The 21-year-old head with $312 gross income qualifies for $40.
 us,scenario_030,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $5,000 of educational assistance and $12,000 of financial assistance in AGI, inflating it from $13,000 to $30,000. With only $13,000 of taxable wages, the single-filer standard deduction eliminates all taxable income, so applying the 10% and 12% brackets to $14,600 was erroneous."
 us,scenario_030,federal_refundable_credits,claude-fable-5,llm_error,age_disability,False,"The model calculated a phased-out childless EITC without first applying the minimum-age requirement. At age 23, the filer is ineligible for the 2026 childless EITC, so the phase-in and phaseout arithmetic never applies."
 us,scenario_030,federal_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated the 23-year-old as eligible for the childless EITC despite the minimum-age requirement. It also mislabeled $1,940 as the childless maximum and submitted that amount even though its own 7.65% calculation produced $994.50."
@@ -1947,36 +2049,38 @@ us,scenario_030,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_outpu
 us,scenario_030,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_030,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model miscalculated Medicare tax: 1.45% of $13,000 is $188.50, not $189 or $189.90. It then compounded the arithmetic error by stating that $806 plus $189 equals $995.90; the exact component sum is $994.50."
 us,scenario_030,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the ordinary 130% FPL gross-income test as controlling and failed to apply Texas TANF non-cash categorical eligibility. That pathway preserves eligibility, after which the one-person minimum allotment produces $287.68 annually."
-us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model denied SNAP under the ordinary gross-income screen and never applied TANF non-cash categorical eligibility at the household's 1.6 gross and 1.27 net income ratios. Its discussion of ABAWD compliance does not replace that eligibility pathway or the minimum-allotment calculation.
-us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly stated that Texas's categorical-income pathway did not cover this household and imposed its own 165% gross-income cutoff. PolicyEngine applies TANF non-cash categorical eligibility, and the resulting benefit is the one-person minimum allotment despite the high expected contribution."
-us,scenario_030,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $12,000 financial assistance from SNAP income and therefore understated net income and the expected contribution. With countable net income around $1,653–$1,658 monthly, the formula falls below zero before the eligible household's minimum allotment is applied, yielding $287.68 annually rather than $2,298."
-us,scenario_030,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model calculated the benefit from wages alone and omitted the countable financial assistance reflected in SNAP net income. Including it raises the expected contribution above the maximum allotment, so categorical eligibility leads only to the annualized minimum benefit of $287.68."
-us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model expressly rejected broad categorical eligibility and applied the ordinary 130% FPL gross-income test. The household instead qualifies through Texas TANF non-cash categorical eligibility and receives the minimum allotment after its expected contribution exceeds the maximum.
-us,scenario_030,snap,claude-sonnet-5,llm_error,other,False,"The model's own arithmetic gives a benefit near $98 monthly, not $350, because $298 minus roughly $200 is roughly $98. It also omitted countable financial assistance, which raises the expected contribution above the maximum and leaves only the $287.68 annual minimum allotment."
-us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income cutoff and also counted educational assistance in its $30,000 total. TANF non-cash categorical eligibility governs, and the eligible household receives the one-person minimum allotment totaling $287.68."
-us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model denied eligibility solely because wages plus financial assistance exceeded the ordinary 130% FPL limit. It omitted the TANF non-cash categorical pathway, under which the household remains eligible and receives the minimum allotment."
-us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $2,604 answer implies an expected contribution based primarily on wages and omits the countable assistance embedded in net income of about $1,653–$1,658 monthly. The full net-income calculation exhausts the maximum allotment, leaving the annualized minimum benefit of $287.68."
-us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated the ordinary 130% FPL gross-income limit as an absolute bar. It failed to apply TANF non-cash categorical eligibility, which permits the one-person household to receive the minimum allotment."
-us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model treated 165% FPL as a disqualifying gross-income ceiling instead of applying the traced TANF non-cash categorical-eligibility tests using both gross and net ratios. The household passes that pathway and receives $287.68 through monthly minimum allotments.
-us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model equated net income high enough to exhaust the formula benefit with complete ineligibility. Categorical eligibility remains intact, and an eligible one-person household receives the minimum allotment rather than zero."
-us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model incorrectly treated the net-income threshold as eliminating SNAP eligibility and payment. TANF non-cash categorical eligibility applies, and the minimum-benefit rule converts the otherwise nonpositive formula amount into $287.68 annually."
-us,scenario_030,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP household member; pregnancy does not increase the SNAP assistance-unit size before birth. It also allowed the elderly-or-disabled medical deduction for a 23-year-old without a listed disability, producing a large benefit instead of the one-person minimum allotment."
-us,scenario_030,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized countable cash income as very low and therefore calculated a substantial formula benefit. Countable net income is about $1,653–$1,658 monthly, making the expected contribution exceed the maximum allotment and leaving only $287.68 in annual minimum benefits."
-us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model returned zero without applying the household's TANF non-cash categorical eligibility or the one-person minimum-allotment rule. Those steps produce positive monthly minimums totaling $287.68.
-us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model correctly recognized that 30% of net income exceeds the maximum allotment but incorrectly converted that result to no payment. Because the household is categorically eligible through TANF non-cash eligibility, the one-person minimum allotment applies and totals $287.68."
-us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model imposed the ordinary one-person gross-income limit and stopped. TANF non-cash categorical eligibility supersedes that denial, and the eligible household receives the minimum allotment."
-us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model correctly excluded educational assistance but wrongly treated wages plus financial assistance above the ordinary gross limit as dispositive. It omitted TANF non-cash categorical eligibility and the subsequent minimum-benefit rule.
-us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model denied SNAP at the ordinary gross-income screen. The household qualifies through TANF non-cash categorical eligibility and receives monthly minimum allotments totaling $287.68.
-us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"The zero answer omits the traced TANF non-cash categorical-eligibility pathway despite assets of only $190. Once eligibility is established, the one-person minimum-allotment rule produces $287.68 annually."
-us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model treated the 130% FPL gross-income test as the final eligibility rule. TANF non-cash categorical eligibility applies at the traced income ratios, so the household receives the minimum benefit rather than zero."
-us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model correctly excluded educational assistance but incorrectly denied the household under the ordinary 130% FPL gross-income limit. It missed TANF non-cash categorical eligibility and the one-person minimum allotment.
-us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"The model both counted excluded educational assistance in gross income and treated the 130% FPL test as an absolute bar. TANF non-cash categorical eligibility applies, after which the minimum-allotment rule yields $287.68."
-us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required structured-output contract."
-us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,The model denied eligibility because wages plus recurring assistance exceeded the ordinary one-person gross limit. It omitted TANF non-cash categorical eligibility and the minimum allotment available after the formula benefit is exhausted.
-us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly framed SNAP as requiring a household of at least two for a pregnant applicant and inconsistently stated that pregnancy counts as two; the SNAP unit remains one person. It also misread annual post-deduction income as exceeding a monthly net limit and omitted TANF non-cash categorical eligibility and the minimum benefit.
-us,scenario_030,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the standard deduction and countable financial assistance from its operative net-income calculation, then abandoned its own $372–$384 result for an unsupported $2,576. The traced net income makes the expected contribution exceed the maximum, so the eligible household receives only $287.68 in annual minimum allotments."
-us,scenario_030,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP member and reduced net income to zero without valid deductions. The correct unit has one member and net income around $1,653–$1,658 monthly, producing the one-person minimum allotment totaling $287.68; its explanation also states $4,272 while submitting $3,576."
+us,scenario_030,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model applied the ordinary 130% FPL gross-income test as a disqualifier and omitted Texas TANF non-cash categorical eligibility. That pathway preserves eligibility and produces the one-person minimum allotment despite the high expected contribution.
+us,scenario_030,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated gross income under the ordinary SNAP test as controlling and never applied TANF non-cash categorical eligibility. Work compliance does not eliminate the minimum allotment available through that categorical pathway.
+us,scenario_030,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that income just above Texas's BBCE threshold made the household ineligible. PolicyEngine's TANF non-cash categorical pathway accepts the 1.6 gross-income and 1.27 net-income ratios, after which the household receives the minimum allotment."
+us,scenario_030,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $12,000 financial assistance from the SNAP benefit computation and consequently understated net income and the expected contribution. The trace produces net income around $1,653–$1,658 and an expected contribution above the maximum allotment, so categorical eligibility yields only the monthly minimum."
+us,scenario_030,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model calculated the benefit from wages alone after only the earned-income and standard deductions, omitting the countable-income treatment that produces net income around $1,653–$1,658. The resulting contribution exceeds the maximum allotment, leaving only the minimum benefit."
+us,scenario_030,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model explicitly denied the applicable broad categorical pathway and made the ordinary 130% gross-income test dispositive. Texas TANF non-cash categorical eligibility instead preserves eligibility at the traced income ratios and triggers the minimum allotment.
+us,scenario_030,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model's own arithmetic gives a benefit near $98 per month, not $350, so its $4,200 annual answer exceeds even the stated $298 one-person monthly maximum. It also omitted the higher traced net income and resulting minimum-allotment floor."
+us,scenario_030,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model summed all listed income and applied the ordinary 130% FPL test without applying TANF non-cash categorical eligibility. The categorical pathway makes the household eligible for the minimum allotment.
+us,scenario_030,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated $25,000 of gross income as automatically disqualifying under the ordinary test. It omitted TANF non-cash categorical eligibility, which permits the household to receive the minimum allotment."
+us,scenario_030,snap,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a substantial benefit based primarily on $13,000 of earnings and omits the traced countable-income calculation. Net income around $1,653–$1,658 makes the expected contribution exceed the maximum allotment, so only the minimum is paid."
+us,scenario_030,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model correctly kept the unborn child outside the SNAP unit but incorrectly made the ordinary gross-income limit dispositive. TANF non-cash categorical eligibility overrides that disqualification and yields the minimum allotment.
+us,scenario_030,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated the 165% FPL figure as a hard cutoff and denied eligibility. The traced gross-income ratio is 1.6 and the household qualifies through TANF non-cash categorical eligibility, producing the minimum benefit."
+us,scenario_030,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model equated net income high enough to reduce the formula benefit to zero with SNAP ineligibility. Categorical eligibility entitles this one-person household to the minimum allotment even when 30% of net income exceeds the maximum allotment.
+us,scenario_030,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model treated the net-income threshold as eliminating any positive allotment. TANF non-cash categorical eligibility preserves eligibility, and the minimum-benefit rule supplies a positive monthly amount."
+us,scenario_030,snap,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP household member and then applied an elderly-or-disabled medical deduction to a 23-year-old with no disability. The SNAP unit has one person, the medical deduction is unavailable, and the traced high contribution leaves the one-person minimum allotment."
+us,scenario_030,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model characterized countable cash income as very low and awarded a substantial benefit, omitting the income calculation that produces monthly net income around $1,653–$1,658. That net income drives the formula below zero, leaving only the minimum allotment through categorical eligibility."
+us,scenario_030,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assigned zero from the one-person composition and $13,000 of earnings without applying TANF non-cash categorical eligibility or the minimum-allotment rule. Those rules produce a positive annual benefit of $287.68."
+us,scenario_030,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model correctly identified that 30% of net income exceeds the maximum allotment but incorrectly converted that result to zero. Because the categorically eligible unit has one member, the minimum-allotment rule applies."
+us,scenario_030,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model made the ordinary one-person gross-income limit dispositive. TANF non-cash categorical eligibility applies at the traced income ratios and preserves entitlement to the minimum allotment.
+us,scenario_030,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model correctly excluded educational assistance but incorrectly denied SNAP solely because wages plus financial assistance exceeded the ordinary gross-income limit. It omitted TANF non-cash categorical eligibility and the resulting minimum benefit.
+us,scenario_030,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated $25,000 of countable gross income as an automatic bar. The TANF non-cash categorical pathway accepts the household's traced income ratios and leads to the one-person minimum allotment."
+us,scenario_030,snap,grok-4.3,llm_error,categorical_eligibility,False,"The zero answer ignores that $190 passes the asset test and that TANF non-cash categorical eligibility preserves SNAP eligibility. With the formula benefit otherwise exhausted, the one-person minimum allotment produces $287.68 annually."
+us,scenario_030,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test as a hard eligibility bar. It omitted TANF non-cash categorical eligibility, under which the household receives the minimum allotment."
+us,scenario_030,snap,grok-4.6,llm_error,categorical_eligibility,False,The model treated both the regular gross-income test and its asserted 165% Texas limit as disqualifying. The trace applies TANF non-cash categorical eligibility at a gross-income ratio of 1.6 and awards the minimum allotment.
+us,scenario_030,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model correctly excluded educational assistance but incorrectly stopped at the ordinary 130% FPL test. TANF non-cash categorical eligibility bypasses that disqualification and produces the minimum benefit.
+us,scenario_030,snap,inkling,llm_error,categorical_eligibility,False,"The model both counted educational assistance in gross income and treated the ordinary 130% FPL test as dispositive. The traced categorical-eligibility pathway qualifies the unit, whose high contribution then results in the minimum allotment."
+us,scenario_030,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable SNAP value or explanation, so it failed the required output contract."
+us,scenario_030,snap,kimi-k3,llm_error,categorical_eligibility,False,The model denied eligibility because wages plus financial assistance exceeded the ordinary one-person gross-income limit. It omitted TANF non-cash categorical eligibility and the minimum-allotment rule.
+us,scenario_030,snap,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly stated that pregnancy either requires a two-person SNAP unit or supports counting the unborn child; the unit remains one person. It also treated the ordinary income limit as dispositive instead of applying TANF non-cash categorical eligibility and the one-person minimum allotment.
+us,scenario_030,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model treated $25,000 of gross income as disqualifying under the ordinary 130% FPL test. TANF non-cash categorical eligibility preserves eligibility and produces the minimum allotment."
+us,scenario_030,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model computed several mutually inconsistent annual amounts and then submitted $2,576 without arithmetic supporting it. It omitted the traced net-income calculation of about $1,653–$1,658; the resulting contribution exceeds the maximum and leaves only the changing monthly minimum allotments totaling $287.68."
+us,scenario_030,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted the unborn child as a second SNAP member and reduced net income to zero through unspecified deductions. SNAP uses a one-person unit here, and the traced net income produces only the one-person minimum allotment; its explanation also states $4,272 while submitting $3,576."
 us,scenario_030,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_030,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no TANF output, violating the required structured-output contract rather than making a substantive benefit calculation."
 us,scenario_030,tanf,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly counted the unborn child as a current dependent child and constructed a two-person Texas TANF assistance unit. A pregnant adult with no born dependent child does not satisfy the family-with-dependent-children requirement, so the $271 monthly grant schedule does not apply."
@@ -1984,34 +2088,36 @@ us,scenario_031,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_er
 us,scenario_031,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model wrongly denied the listed alimony deduction and used an estimated $10,207 post-TCJA-reversion standard deduction. PolicyEngine deducts the $1,165 alimony expense and applies the applicable 2026 single age-65-or-older standard deduction, leaving no taxable income."
 us,scenario_031,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced zero taxable income, zero tax, and zero nonrefundable credits, but it submitted $1,268.50. Its numeric output directly contradicts every computation step in its explanation."
 us,scenario_031,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $5,357.50 of Social Security as taxable even though provisional income is below the $25,000 single-filer base, and it then failed to follow its own calculation that the elderly credit reduced tax to zero. The alimony adjustment yields $13,243 of AGI, the standard deduction eliminates taxable income, and the submitted $2,439 has no support in its stated arithmetic."
-us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model applied a MAGI-style calculation and compared it with a 138% FPL threshold. It omitted the optional senior-or-disabled pathway's deduction for health insurance premiums, including Medicare Part B, which brings countable income within the California limit."
-us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted alimony but omitted the non-MAGI deduction for health insurance premiums, including Medicare Part B. It also treated the $4,200 bank balance as a potential disqualifier even though the head passes the applicable asset test."
-us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly described California's aged-and-disabled determination as MAGI-based and compared gross retirement income with a 100% FPL limit. The applicable non-MAGI pathway deducts Medicare Part B and other health insurance premiums from countable income, producing income eligibility."
-us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model compared the full $23,853 of retirement income directly with an aged-program FPL threshold. It failed to deduct health insurance premiums, including Medicare Part B, under the optional senior-or-disabled income test."
-us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,The model repeatedly evaluated the head under MAGI and gross-income FPL comparisons instead of completing the non-MAGI senior-or-disabled calculation. That calculation deducts Medicare Part B and other health insurance premiums from countable income and yields eligibility.
-us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model substituted an SSI payment-income standard for California's optional senior-or-disabled income test. SSI receipt is not required, and the applicable calculation deducts health insurance premiums, including Medicare Part B, leaving the head income-eligible."
-us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model compared unadjusted monthly income with a 138% FPL figure. It omitted the senior-or-disabled pathway's health-insurance-premium deduction, including Medicare Part B, which reduces countable income below the applicable limit."
-us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used the full $23,853 as countable income against a 138% FPL limit. The non-MAGI senior-or-disabled calculation deducts Medicare Part B and other health insurance premiums and passes the income test."
-us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a direct comparison of gross household income with a general Medi-Cal limit. The applicable senior-or-disabled pathway deducts health insurance premiums, including Medicare Part B, and the resulting countable income satisfies its limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated $23,853 as the income tested against a 138% FPL aged-program threshold. It omitted the non-MAGI deduction for Medicare Part B and other health insurance premiums that makes the head income-eligible."
-us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model concluded that monthly income exceeded 138% FPL without applying the senior-or-disabled income-counting rules. Deducting health insurance premiums, including Medicare Part B, produces countable income within the applicable limit."
-us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,The model compared annual gross retirement income with a generic senior Medi-Cal threshold. It omitted the optional senior-or-disabled pathway's Medicare Part B and other health-insurance-premium deductions.
-us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model asserted that income exceeded the non-MAGI limit but did not calculate non-MAGI countable income. The required deduction for health insurance premiums, including Medicare Part B, makes the head pass that income test."
-us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare eligibility as precluding Medicaid. Dual Medicare-Medicaid eligibility is allowed, and the head qualifies through California's optional senior-or-disabled Medicaid category."
-us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an additional disability or low-income flag and failed to recognize that age 67 itself satisfies the aged-or-disabled categorical condition. It also omitted the pathway's health-insurance-premium deduction, after which the head passes the income test."
-us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,taxable_income_or_deductions,False,The model invoked an aged Medi-Cal income or spend-down standard while counting only the small listed over-the-counter expense as medical spending. It omitted the deductible Medicare Part B premium and other health insurance premiums used by the optional senior-or-disabled income test.
-us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The answer implies that gross retirement income was compared directly with the California aged-pathway limit. The required Medicare Part B and other health-insurance-premium deductions reduce countable income enough to satisfy that test.
-us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model treated annual retirement income as exceeding the aged-pathway limit without applying its income deductions. Deducting health insurance premiums, including Medicare Part B, yields qualifying countable income."
-us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied an SSI-linked income level as the controlling threshold. The head instead qualifies under California's optional senior-or-disabled category, whose countable-income test deducts Medicare Part B and other health insurance premiums."
-us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Medicare eligibility excludes full Medicaid dual eligibility. California's senior-or-disabled pathway permits dual eligibility, and this head passes its income and asset tests."
-us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"The model treated both $23,853 of gross income and $4,200 of bank assets as disqualifying. The head passes the applicable asset test, and the non-MAGI income calculation deducts health insurance premiums, including Medicare Part B, to produce income eligibility."
-us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"The model imposed a $2,000 aged-Medi-Cal asset limit and rejected the $4,200 bank balance. Under the applicable California optional senior-or-disabled pathway, the head satisfies the asset test."
-us,scenario_031,head_medicaid_eligible,inkling,llm_error,taxable_income_or_deductions,False,"The model compared the full $23,853 with a 100% FPL threshold. It omitted the senior-or-disabled pathway's deduction for health insurance premiums, including Medicare Part B, which makes countable income eligible."
-us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model incorrectly required disability, blindness, pregnancy, or SSI receipt after excluding the head from the under-65 expansion group. Age 67 independently satisfies the aged-or-disabled categorical condition for California's optional non-MAGI pathway; SSI receipt is not required."
-us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,The model recognized the aged pathway but concluded that retirement income exceeded its limit without applying the pathway's countable-income rules. Medicare Part B and other health insurance premiums are deductible and bring countable income within the limit.
-us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model relied on the roughly $24,000 gross-income total and stopped short of the applicable countable-income calculation. California's optional senior-or-disabled test deducts health insurance premiums, including Medicare Part B, and yields income eligibility."
-us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated age 67 and Medicare eligibility as reasons to look only for disability, nursing-home, or SSI status. Age 67 itself establishes the aged category, SSI receipt is unnecessary, and the non-MAGI premium deductions make the head income-eligible."
-us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that the 67-year-old lacked an eligibility category because disability, blindness, and pregnancy were absent. Senior status supplies the categorical basis, and the optional senior-or-disabled calculation deducts Medicare Part B and other health insurance premiums before applying the income limit."
+us,scenario_031,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model applied a MAGI-style alimony adjustment and compared the result with a 138% FPL limit. The senior-or-disabled non-MAGI calculation instead deducts health-insurance premiums, including Medicare Part B, and that calculation places the head within the applicable income limit."
+us,scenario_031,head_medicaid_eligible,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model compared gross retirement income, less only alimony, with a generic 138% FPL threshold and treated $4,200 of assets as disqualifying. The optional senior-or-disabled pathway deducts Medicare Part B premiums from countable income, and the head passes both its income and asset tests."
+us,scenario_031,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly described the aged-and-disabled determination as MAGI-based and tested the full $23,853 against a 100% FPL threshold. California's applicable senior-or-disabled pathway is non-MAGI and deducts Medicare Part B premiums, producing qualifying countable income."
+us,scenario_031,head_medicaid_eligible,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the full $23,853 as countable income and compared it with a broad 100%-138% FPL range. It omitted the senior-or-disabled pathway's deduction for health-insurance premiums, including Medicare Part B, which makes the head income-eligible."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model repeatedly substituted MAGI and gross-income comparisons for the applicable non-MAGI senior-or-disabled calculation. It never deducted Medicare Part B premiums from countable income, so it incorrectly found the head above the pathway's limit."
+us,scenario_031,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an SSI cash-benefit income standard of about $13,596 as the controlling Medicaid limit. The head qualifies through California's optional senior-or-disabled Medicaid category without receiving SSI, and its separate income test deducts Medicare Part B premiums."
+us,scenario_031,head_medicaid_eligible,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model compared unadjusted monthly gross income of $1,987.75 with an estimated 138% FPL limit. The non-MAGI senior-or-disabled test first deducts Medicare Part B premiums, after which the head satisfies the applicable income limit."
+us,scenario_031,head_medicaid_eligible,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model compared the full $23,853 of retirement income directly with 138% FPL. It omitted the Medicare Part B premium deduction required by the senior-or-disabled non-MAGI income calculation."
+us,scenario_031,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,The answer implies a gross-household-income comparison with a generic Medi-Cal limit. The applicable senior-or-disabled calculation deducts Medicare Part B premiums from countable income and yields income eligibility.
+us,scenario_031,head_medicaid_eligible,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated $23,853 as the income tested against a 138% FPL threshold. It omitted the non-MAGI senior-or-disabled pathway's Medicare Part B premium deduction, which brings countable income within the limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,The model concluded that monthly income exceeded 138% FPL without applying the pathway's income-counting rules. Deducting Medicare Part B premiums under the senior-or-disabled non-MAGI test produces qualifying countable income.
+us,scenario_031,head_medicaid_eligible,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the full $23,853 as the decisive annual income amount. The applicable aged pathway deducts health-insurance premiums, including Medicare Part B, and the resulting countable income passes its limit."
+us,scenario_031,head_medicaid_eligible,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model asserted that income exceeded the elderly non-MAGI limit but omitted the controlling premium deduction. California's senior-or-disabled test deducts Medicare Part B premiums, leaving the head income-eligible."
+us,scenario_031,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated Medicare eligibility as excluding Medicaid eligibility. Medicare beneficiaries can qualify concurrently for Medicaid, and this head qualifies through California's optional senior-or-disabled category."
+us,scenario_031,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required a separately listed disability or low-income flag and overlooked that age 67 itself satisfies the aged-or-disabled categorical condition. It also failed to apply the pathway's Medicare Part B premium deduction, under which the head passes the income test."
+us,scenario_031,head_medicaid_eligible,gpt-5.5,llm_error,health_coverage,False,"The model treated the listed $50 over-the-counter expense as the only relevant medical deduction and found income above the aged or spend-down standard. The senior-or-disabled pathway deducts health-insurance premiums, specifically including Medicare Part B, and that deduction establishes income eligibility."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The answer implies that gross retirement income was compared directly with California's aged Medicaid threshold. The required Medicare Part B premium deduction reduces non-MAGI countable income enough to pass the senior-or-disabled test.
+us,scenario_031,head_medicaid_eligible,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model treated annual retirement income as exceeding the aged-pathway limit without performing the pathway's premium deduction. After deducting Medicare Part B premiums, the head satisfies the senior-or-disabled income test."
+us,scenario_031,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model substituted an SSI-linked income level for California's optional senior-or-disabled Medicaid income test. That separate test deducts Medicare Part B premiums and is satisfied even though the head receives no SSI.
+us,scenario_031,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model invented an exclusion of full dual eligibles from PolicyEngine's Medicaid rules. Medicare eligibility does not bar Medicaid, and the head independently meets California's senior-or-disabled income and asset conditions."
+us,scenario_031,head_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,"The model incorrectly treated both $23,853 of gross income and $4,200 of bank assets as failures. The head passes the pathway's asset test, while its non-MAGI income test deducts Medicare Part B premiums and also yields eligibility."
+us,scenario_031,head_medicaid_eligible,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model conflated MAGI, Aged and Disabled FPL, and Medicare Savings Program tests and counted the full $23,853. The applicable senior-or-disabled Medicaid test deducts Medicare Part B premiums and produces income eligibility."
+us,scenario_031,head_medicaid_eligible,grok-build-0.1,llm_error,asset_resource,False,"The model applied a $2,000 asset limit and treated the $4,200 bank balance as disqualifying. Under the applicable California senior-or-disabled pathway, the head satisfies the asset test."
+us,scenario_031,head_medicaid_eligible,inkling,llm_error,taxable_income_or_deductions,False,"The model compared approximately $23,853 of gross income with a 100% FPL limit. It omitted the senior-or-disabled non-MAGI deduction for Medicare Part B premiums, under which the head passes the applicable income test."
+us,scenario_031,head_medicaid_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model overlooked that being age 67 independently satisfies the aged-or-disabled categorical condition; disability, blindness, pregnancy, and SSI receipt are not required. The head therefore enters the optional senior-or-disabled pathway, where the premium-adjusted income and assets pass."
+us,scenario_031,head_medicaid_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the head from the under-65 expansion group but then treated listed retirement income as exceeding the aged Medi-Cal limit. It omitted the senior-or-disabled pathway's deduction for Medicare Part B premiums, which produces qualifying countable income."
+us,scenario_031,head_medicaid_eligible,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model stopped at an approximate $24,000 gross-income comparison and did not calculate non-MAGI countable income. The required Medicare Part B premium deduction places the head within the senior-or-disabled income limit."
+us,scenario_031,head_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"The model imposed a $2,000 SSI resource limit and counted the $4,200 bank balance as a Medicaid failure. The head passes California's applicable senior-or-disabled asset test, and Medicare Part B premiums reduce countable income enough to pass its income test as well."
+us,scenario_031,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated Medicare eligibility and absence of disability or SSI receipt as barriers to Medicaid. Age 67 supplies the aged categorical basis, and the optional senior-or-disabled pathway deducts Medicare Part B premiums before finding the head income-eligible."
+us,scenario_031,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that the 67-year-old lacked a Medicaid eligibility category. Age alone establishes the senior-or-disabled category, whose premium-adjusted income test and asset test the head satisfies."
 us,scenario_031,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated income below California's 200% FPL broad-based categorical-eligibility threshold as guaranteeing SNAP eligibility and the $23 monthly minimum. The household fails the countable-net-income benefit determination, and the minimum allotment applies only after eligibility is established."
 us,scenario_031,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly concluded that broad-based categorical eligibility guarantees the $23 monthly minimum even when 30% of net income exceeds the maximum allotment. That income calculation yields no eligibility for a positive allotment, so the minimum-benefit rule does not apply."
 us,scenario_031,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that countable income was very low without applying the household's $23,853 annual retirement, IRA, and pension income to the SNAP net-income calculation. The resulting countable income produces no SNAP benefit, rather than the $144 monthly award implicit in its answer."
@@ -2052,104 +2158,113 @@ us,scenario_032,child1_wic_eligible,qwen3.8-max,llm_error,categorical_eligibilit
 us,scenario_032,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed that the TCJA rules expired for 2026, substituted a roughly $16,150 MFJ standard deduction and a $1,000 CTC, and thereby produced $1,477 of pre-credit tax minus $1,000. Under the applicable 2026 parameters, the $2,200 CTC supplies $500 of nonrefundable credit, which fully reduces the liability to $0; its remaining $1,700 and the EITC are refundable and do not enter this output."
 us,scenario_032,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits."
 us,scenario_032,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for the requested output. The required computation applies $500 of nonrefundable CTC to eliminate the positive pre-credit liability, yielding $0 before refundable credits."
-us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model rounded the 2026 one-child EITC down to $4,420 instead of applying the exact $4,427 amount. Adding the correctly identified $1,700 refundable CTC yields $6,127, not $6,120."
-us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated the household as having three EITC qualifying children even though the spouse is not a dependent child and only the age-6 child qualifies. It also used a $1,600 refundable CTC instead of $1,700 and failed to compute the applicable $4,427 one-child EITC."
-us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used inconsistent estimated EITC figures and then added an unexplained $215 “estimate refinement.” The applicable EITC is $4,427, which combines with the $1,700 refundable CTC for $6,127."
-us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly placed $29,000 of earnings in the one-child EITC phaseout and reduced the credit to $3,372. The engine calculation produces a $4,427 EITC, and the correctly calculated $1,700 refundable CTC brings the total to $6,127."
-us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model calculated an inapplicable $3,412 EITC and then inserted an unexplained additional $1,100 into refundable credits. Only the $4,427 EITC and $1,700 refundable CTC apply."
-us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied the single-filer $34,000 Social Security provisional-income threshold to a joint return, manufactured $2,557 of taxable benefits, and then treated unused nonrefundable CTC as refundable above the $1,700 refundable cap. The refundable components are $4,427 of EITC and $1,700 of refundable CTC."
-us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented an “expanded family credit” for the spouse after correctly recognizing that the spouse is not a qualifying child or dependent. No such additional refundable credit applies; the total contains only the $4,427 EITC and $1,700 refundable CTC."
-us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the wrong EITC amount and obsolete refundable-CTC rules—a $3,000 earnings threshold and $1,000 cap. The applicable amounts are a $4,427 EITC and a $1,700 refundable CTC."
-us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model reduced the one-child EITC to $3,599.34 and capped the refundable CTC at $1,000. The 2026 computation instead yields $4,427 and $1,700, respectively."
-us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model supplied no component calculation and understated the combined credits by $665. The applicable computation is $4,427 of EITC plus $1,700 of refundable CTC, totaling $6,127."
-us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA $1,000 Additional Child Tax Credit cap instead of the applicable $1,700 refundable CTC. Its EITC estimate was also $13 too high; the correct component is $4,427."
-us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an understated $4,014 EITC and an obsolete $1,000 refundable-CTC cap. The applicable components are $4,427 and $1,700."
-us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration restored a $1,000 CTC and a $3,000 ACTC earnings threshold for 2026. The refundable CTC is $1,700, while the EITC is $4,427 rather than $4,428."
-us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model named the correct two credit categories but did not calculate either component and understated their sum by $879. Their actual amounts are $4,427 for EITC and $1,700 for refundable CTC."
-us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_refundable_credits, violating the required output contract."
-us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize that the wages and qualifying age-6 child generate both EITC and refundable CTC eligibility. Those credits are $4,427 and $1,700, not zero."
-us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required refundable-credit amounts to be explicitly supplied rather than calculating them from wages, filing status, and the qualifying child. Those facts produce a $4,427 EITC and $1,700 refundable CTC."
-us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $1,800 refundable-CTC limit instead of the applicable $1,700 cap. Its $4,427 EITC component was correct, so the total is $6,127 rather than $6,227."
-us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model identified the two applicable credits but understated their combined amount by $80 without providing a component calculation. The exact components are a $4,427 EITC and $1,700 refundable CTC."
-us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,other,False,"The model named EITC and CTC but assigned an unsupported $2,500 aggregate that omits $3,627 of the calculated benefits. The two refundable components are $4,427 and $1,700."
-us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied an obsolete $1,000 refundable-CTC cap and overstated the EITC by $9. The applicable amounts are $1,700 and $4,427."
-us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a $1,000 refundable-CTC cap instead of $1,700 and overstated the EITC by $12. The correct component sum is $4,427 plus $1,700."
-us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly used the $1,700 refundable-CTC cap but overstated the EITC by $22 because it used an estimated $4,449 maximum. The engine applies the exact $4,427 EITC."
-us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_refundable_credits, violating the required output contract."
-us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly classified the household as having two EITC qualifying children and then omitted the refundable CTC from its submitted total despite discussing it. Only one child qualifies, producing a $4,427 EITC plus a $1,700 refundable CTC."
-us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model cycled through several incorrect EITC maxima, phaseout thresholds, and refundable-CTC caps before submitting unsupported components of $4,509 and $800. The applicable fixed components are $4,427 of EITC and $1,700 of refundable CTC."
-us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model drastically understated both credits, assigning $1,171 to EITC and $937 to the refundable CTC without a supporting calculation. The 2026 rules yield $4,427 and $1,700, respectively."
-us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model treated the federal 130% FPL test and SNAP/TANF direct certification as the only qualifying pathways. It omitted Minnesota's universal free school meals program, which makes the K-12 child eligible at the household's 1.56 FPL ratio."
-us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied the National School Lunch Program's 130% FPL income threshold as dispositive after calculating $42,664 of household income. It failed to apply Minnesota's universal free school meals program, which extends free meals beyond that federal income threshold."
-us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,"The model denied eligibility solely because $42,664 exceeded its 130% FPL threshold. It omitted Minnesota's universal free school meals program, under which the enrolled K-12 child qualifies regardless of this income-test result."
-us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model correctly totaled wages and Social Security benefits to $42,664 but incorrectly stopped at the federal 130% FPL test. Minnesota's universal free school meals program supplies the qualifying pathway despite the household's 1.56 FPL ratio."
-us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed that absent SNAP/TANF receipt or an explicitly listed free-meal status, no free-meal pathway existed. It failed to derive eligibility from the listed Minnesota residence and Minnesota's universal free school meals program."
-us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model treated income below the federal limit or categorical status as necessary for free meals. It omitted Minnesota's universal program, which makes the household's K-12 child eligible even though income is 1.56 times the poverty guideline and categorical eligibility is false."
-us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model concluded that income above the federal free-meal threshold prevented positive support. Minnesota's universal free school meals rule overrides that income-based denial for the enrolled K-12 child.
-us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model limited eligibility to the federal income threshold and direct certification. It omitted Minnesota's universal free school meals pathway, which produces positive annual free-meal support without either condition."
-us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model correctly recognized that household income was about 156% of the poverty guideline but incorrectly treated the 130% federal limit as controlling. Minnesota's universal free school meals program makes the K-12 child eligible above that limit.
-us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model used the household's roughly 156.7% FPL ratio to deny eligibility under the 130% federal threshold. It failed to apply Minnesota's universal free school meals program, the decisive state pathway."
-us,scenario_032,free_school_meals_eligible,inkling,llm_error,state_local_rule,False,"The model denied free meals solely because household income exceeded 130% FPL. It omitted Minnesota's universal free school meals program, which extends free eligibility to the enrolled child regardless of that threshold result."
-us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for the requested output. It therefore failed the required structured-output contract rather than performing the Minnesota universal free school meals eligibility determination.
-us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,state_local_rule,False,"The model applied only PolicyEngine's federal 130% FPL income test and denied eligibility at about 156% FPL. It omitted Minnesota's universal free school meals program, which independently sets the household's school-meal tier to FREE."
+us,scenario_032,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model rounded the $4,427 EITC down to $4,420 despite identifying the correct $1,700 refundable CTC. Using the exact EITC parameter gives $4,427 + $1,700 = $6,127."
+us,scenario_032,federal_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly described the spouse and child as three qualifying dependents/children and then applied an unsupported aggregate phaseout. Only the age-6 child is the qualifying child, producing a $4,427 EITC and $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated EITC parameters, calculated $4,328, and then added an unexplained $215 adjustment. The applicable EITC is $4,427, so no extra refinement belongs on top of the $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model placed $29,000 in the one-child joint-filer EITC phaseout and reduced the credit to $3,372. The applicable calculation yields the full $4,427 EITC, which combines with the $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model calculated an inapplicable $3,412 EITC and then introduced an unexplained $1,100 refundable amount beyond the stated $1,700 child credit. Refundable credits here contain only the $4,427 EITC and $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated part of the Social Security dependent benefits as taxable, used that manufactured tax liability to reduce the refundable CTC below its $1,700 cap, and also substituted estimated EITC parameters. The trace calculation uses a $4,427 EITC and the capped $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented an additional refundable credit for the 18-year-old spouse under nonexistent expanded-family-credit provisions. A spouse is not an additional qualifying dependent for such a credit, and the only components are the $4,427 EITC and $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC cap and overstated the EITC as $4,488. The 2026 amounts are $1,700 and $4,427 respectively."
+us,scenario_032,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $1,000 refundable CTC cap and reduced the EITC to $3,599.34 under the wrong parameters. The applicable components are $1,700 of refundable CTC and $4,427 of EITC."
+us,scenario_032,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"Its unexplained $5,462 total omits $665 from the traced $4,427 EITC plus $1,700 refundable CTC calculation. The answer reflects neither the applicable component amounts nor their correct sum."
+us,scenario_032,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied the obsolete pre-TCJA $1,000 Additional Child Tax Credit cap. The refundable CTC is $1,700, and the exact EITC is $4,427 rather than the approximated $4,440."
+us,scenario_032,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 refundable CTC and understated the EITC as $4,014. The correct component values are $1,700 and $4,427."
+us,scenario_032,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model assumed TCJA expiration restored a $1,000 CTC cap and a $3,000 earnings threshold. The applicable refundable CTC is $1,700, while the one-child EITC is $4,427."
+us,scenario_032,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The unexplained $5,248 total understates the traced sum by $879. Applying the $4,427 EITC and $1,700 refundable CTC yields $6,127."
+us,scenario_032,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value for federal_refundable_credits, violating the required output contract."
+us,scenario_032,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize both refundable-credit pathways despite $29,000 of earnings and one qualifying child. Those facts generate a $4,427 EITC and a $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required refundable credit amounts to be explicitly supplied instead of calculating statutory eligibility from wages and the qualifying child. The household qualifies for the $4,427 EITC and $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an $1,800 refundable child tax credit limit. The applicable cap is $1,700, so $4,427 plus $1,700 equals $6,127."
+us,scenario_032,federal_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model recognized the two correct credit categories but understated their sum by $80. Their traced values are $4,427 for EITC and $1,700 for refundable CTC."
+us,scenario_032,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The unsupported $2,500 estimate substantially undercounts both refundable components. The one-child calculation produces $4,427 of EITC plus $1,700 of refundable CTC."
+us,scenario_032,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied the obsolete post-sunset $1,000 refundable CTC cap and approximated EITC at $4,436. The applicable amounts are a $1,700 refundable CTC and $4,427 EITC."
+us,scenario_032,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used an obsolete $1,000 Additional CTC cap and understated EITC as $4,318. The correct 2026 components are $1,700 and $4,427."
+us,scenario_032,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model capped the refundable CTC at the obsolete $1,000 amount. It also overstated EITC by $12; the applicable values are $1,700 and $4,427."
+us,scenario_032,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly used the $1,700 refundable CTC but overstated the EITC by $22 through approximate inflation indexing. The exact EITC is $4,427."
+us,scenario_032,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value for federal_refundable_credits, violating the required output contract."
+us,scenario_032,federal_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model incorrectly calculated EITC using two qualifying children and then omitted the refundable CTC from its submitted total despite mentioning it. There is one qualifying child, yielding a $4,427 EITC and $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model applied a roughly $22,190 EITC phaseout threshold that does not govern this married-joint one-child calculation, reducing EITC to $3,348. The applicable EITC is $4,427, alongside the correctly identified $1,700 refundable CTC."
+us,scenario_032,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model cycled through incorrect EITC maxima, phaseout thresholds, and refundable CTC caps before submitting an unsupported $4,509 EITC and $800 ACTC. The applicable fixed components are $4,427 and $1,700."
+us,scenario_032,federal_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model severely understated both components as $1,171 of EITC and $937 of refundable CTC. The traced one-child amounts are $4,427 and $1,700."
+us,scenario_032,free_school_meals_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated the federal 130% FPG test and SNAP/TANF direct certification as the only free-meal pathways. It omitted Minnesota's universal free school meals program, which covers the enrolled K-12 child despite the household's 1.56 FPG ratio."
+us,scenario_032,free_school_meals_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model applied only the National School Lunch Program's 130% FPG income limit and concluded that income above that limit precluded free meals. Minnesota's universal free school meals program independently sets the child's meal tier to FREE.
+us,scenario_032,free_school_meals_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model stopped after comparing household income with the federal 130% FPG threshold. It failed to apply Minnesota's universal free school meals pathway, under which the enrolled K-12 child qualifies without satisfying that income limit."
+us,scenario_032,free_school_meals_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model treated income above 130% FPG as dispositive. It omitted Minnesota's universal free school meals program, which makes the school-age child eligible independently of household income."
+us,scenario_032,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model inferred ineligibility from the absence of SNAP/TANF categorical eligibility and failed to evaluate the state-based pathway. Minnesota's universal program provides free meals to the enrolled K-12 child even without SNAP or TANF.
+us,scenario_032,free_school_meals_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model limited eligibility to the federal income test or an indicated categorical status. It missed Minnesota's universal free school meals program, which independently qualifies the enrolled child."
+us,scenario_032,free_school_meals_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly made the federal income threshold controlling. Minnesota's universal free school meals program overrides the need to meet that income test and produces positive free-meal support for the enrolled child.
+us,scenario_032,free_school_meals_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model considered only the federal income threshold and direct certification. It omitted Minnesota's universal free school meals pathway, which qualifies the K-12 child even though neither of those federal pathways applies."
+us,scenario_032,free_school_meals_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model correctly identified income at about 156% of FPG but incorrectly treated the 130% federal cutoff as exhaustive. Minnesota's universal free school meals program supplies the decisive eligibility pathway.
+us,scenario_032,free_school_meals_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model treated the 130% FPG test and SNAP/TANF categorical eligibility as the complete PolicyEngine rule. It failed to apply Minnesota's universal program, which causes PolicyEngine to return positive free school meal support."
+us,scenario_032,free_school_meals_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model stopped at the finding that household income exceeded 130% FPG. It omitted Minnesota's universal free school meals program, under which the enrolled K-12 child qualifies regardless of that ratio."
+us,scenario_032,free_school_meals_eligible,inkling,llm_error,categorical_eligibility,False,The model used only the federal 130% FPG income test. Minnesota's universal free school meals program independently makes the enrolled child eligible for free meals.
+us,scenario_032,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer or explanation for the requested output. It therefore failed the required output contract rather than completing the Minnesota universal free-meals eligibility calculation.
+us,scenario_032,free_school_meals_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model applied only the federal school-meal income test and treated income at 156% of FPG as disqualifying. It omitted Minnesota's universal free school meals pathway, which sets the enrolled child's tier to FREE."
+us,scenario_032,free_school_meals_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model limited the analysis to the 130% FPG cutoff and SNAP/TANF categorical eligibility. It missed Minnesota's universal free school meals program, which qualifies the K-12 child even though income and categorical tests do not."
 us,scenario_032,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits and tested only $29,000 of wages against 138% FPL. The Medicaid MAGI calculation yields 1.56 times FPL, so the head is above the adult MAGI limit and has no qualifying category."
-us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model acknowledged income near $42,664 but invented a higher Minnesota Medicaid threshold for parents or caretakers. The head’s MAGI is 1.56 times FPL, and the presence of a child does not place the head in a qualifying Medicaid category at that income."
-us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits and treated wages alone as Medicaid MAGI. The engine’s MAGI calculation is 1.56 times FPL, above the expansion-adult limit, leaving the head with category NONE."
-us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model invented a 275% FPL Medicaid threshold for Minnesota parents or caretaker relatives and used it after recognizing that income exceeded 138% FPL. No such eligibility pathway applies here: at 1.56 times FPL, the head qualifies through none of Minnesota’s Medicaid categories."
-us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly counted an income item as a fourth household member, inflating household size from three to four and therefore inflating the poverty threshold. With the correct Medicaid household computation, MAGI is 1.56 times FPL and exceeds the adult limit."
-us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied a generic low-income shortcut without testing the head against Minnesota’s specific Medicaid pathways. The correct comparison places MAGI at 1.56 times FPL, above the applicable adult limit, with no alternate qualifying category."
-us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that household MAGI fell within Minnesota’s expansion range, but the Medicaid MAGI calculation equals 1.56 times FPL. That exceeds the applicable adult threshold, and the head has no other eligibility category."
-us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,"The model substituted a generic low-income characterization for the required income-threshold and categorical tests. MAGI is 1.56 times FPL, above the adult limit, and no other Medicaid pathway applies."
-us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model calculated MAGI as $30,916 and 113% FPL, understating the income measure used by the engine. The correct Medicaid MAGI is 1.56 times FPL, which exceeds the 138% expansion threshold."
-us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated the head’s Medicaid income level as about 113.6% FPL. The engine’s MAGI calculation is 1.56 times FPL, above the adult expansion limit, and the head has no alternate qualifying category."
-us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required output contract rather than completing the Medicaid eligibility calculation.
-us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model treated the household as being at the 138% expansion boundary. The correct Medicaid MAGI level is 1.56 times FPL, above that boundary, and being under age 65 and ineligible for Medicare does not create another Medicaid pathway."
-us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model discarded the Social Security dependent benefits from the head’s Medicaid MAGI and compared wages alone with 138% FPL. The correct MAGI computation produces 1.56 times FPL, so the head exceeds the adult limit and qualifies through no other category."
+us,scenario_032,head_medicaid_eligible,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits from the Medicaid MAGI calculation and therefore compared only $29,000 of wages with 138% FPL. The applicable MAGI computation is 1.56 times FPL, above the adult limit, and the head has no other qualifying category."
+us,scenario_032,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model acknowledged that including Social Security produces income above 138% FPL but then invented a higher Minnesota parent/caretaker pathway. The head qualifies through no parent, caretaker, or other Medicaid category at the traced 1.56-FPL MAGI, so the assigned category is NONE."
+us,scenario_032,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the Social Security dependent benefits and treated wages alone as Medicaid MAGI. The traced MAGI is 1.56 times FPL, not below the 138% adult threshold, and no alternate category applies."
+us,scenario_032,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model invented a 275%-FPL Medicaid threshold for Minnesota parents or caretaker relatives after recognizing that total income exceeds 138% FPL. No such qualifying pathway applies to the head in this computation; at 1.56 times FPL the engine assigns Medicaid category NONE.
+us,scenario_032,head_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a household size of four even though the household contains three people; an income item is not an additional household member. That inflated the poverty threshold and obscured the traced MAGI ratio of 1.56 FPL, which is above the applicable adult limit."
+us,scenario_032,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model asserted low-income eligibility without applying the Minnesota adult Medicaid threshold. The head's MAGI is 1.56 times FPL, above the applicable MAGI limits, and the head qualifies through no other category."
+us,scenario_032,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model placed household MAGI inside Minnesota's adult expansion range without performing the controlling comparison. The traced MAGI equals 1.56 FPL, exceeding the applicable adult limit, and the eligibility category is NONE."
+us,scenario_032,head_medicaid_eligible,grok-4.3,llm_error,thresholds_rates,False,The model substituted a generic low-income characterization for the required eligibility test. At 1.56 FPL the head exceeds the applicable adult Medicaid income limits and has no categorical pathway to eligibility.
+us,scenario_032,head_medicaid_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model computed MAGI as $30,916 and 113% FPL, omitting income included in the engine's Medicaid MAGI computation. The correct computation places MAGI at 1.56 FPL, above Minnesota's 138% expansion-adult limit."
+us,scenario_032,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated the head's Medicaid income as approximately 113.6% FPL. The traced MAGI is 1.56 FPL, so the head is above the 138% adult limit and has no alternate eligibility category."
+us,scenario_032,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value for the requested output. It therefore failed the required structured-output contract before any substantive Medicaid determination could be evaluated.
+us,scenario_032,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model treated the household as being at the Medicaid expansion cutoff and concluded eligibility. The actual MAGI ratio is 1.56 FPL, above the applicable adult threshold, and being under age 65 and off Medicare does not create another Medicaid category."
+us,scenario_032,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used only about $29,000 to $30,916 as MAGI and derived a poverty ratio of 109% to 116%. The controlling MAGI calculation yields 1.56 FPL, above the 138% expansion-adult limit, with no other qualifying pathway."
+us,scenario_032,head_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model recognized that $42,664 exceeds the expansion threshold but then excluded the Social Security dependent benefits by treating them as benefits received by the child, despite the facts assigning them to the head. The traced Medicaid MAGI is 1.56 FPL, and the head consequently qualifies through no Medicaid category."
 us,scenario_032,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_032,head_wic_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model invented a household-level “WIC-eligible context” pathway and transferred eligibility to the head based on living with a child. The head is age 50, is not pregnant, postpartum, or breastfeeding, and therefore fails WIC's person-specific categorical requirement; moreover, the only child is age 6 and is not categorically eligible either."
 us,scenario_032,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly asserted that Minnesota has no mandatory employee state payroll tax in 2026. It omitted the $127.60 Minnesota paid-leave contribution on $29,000 of taxable wages."
-us,scenario_032,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly excluded Minnesota's $127.60 paid-leave employee contribution. It then submitted $2,175.90 even though its own federal FICA calculation was $2,218.50, introducing an additional unsupported arithmetic adjustment."
-us,scenario_032,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model incorrectly asserted that Minnesota has no mandatory employee state payroll tax. It stopped at $2,218.50 of federal FICA and omitted the $127.60 Minnesota paid-leave contribution."
-us,scenario_032,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly treated federal FICA as the complete payroll-tax liability. Minnesota's 2026 paid-leave requirement adds a $127.60 employee contribution on the same $29,000 wage base."
-us,scenario_032,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model incorrectly stated that no mandatory Minnesota employee payroll tax applies. It omitted the $127.60 Minnesota paid-leave contribution from its otherwise correct federal FICA calculation.
-us,scenario_032,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly asserted that Minnesota imposes no separate employee-side payroll tax in 2026. It failed to add the $127.60 Minnesota paid-leave contribution assessed on $29,000 of taxable wages."
-us,scenario_032,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly excluded Minnesota's 2026 paid-family-and-medical-leave employee contribution. That state component is $127.60, so federal FICA of $2,218.50 is not the complete payroll-tax output."
-us,scenario_032,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model recognized the Minnesota paid-leave contribution but applied a 0.35% employee rate, producing $101.50. The applicable contribution in this computation is $127.60 on $29,000 of Minnesota paid-leave taxable wages."
-us,scenario_032,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,The model calculated only the 7.65% federal FICA charge. It omitted Minnesota's mandatory $127.60 paid-leave employee contribution.
-us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model included only Social Security and Medicare taxes and rounded their $2,218.50 sum to $2,219. It omitted the separate $127.60 Minnesota paid-leave contribution."
+us,scenario_032,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model omitted Minnesota's $127.60 employee paid-leave contribution by asserting that the state has no mandatory employee payroll tax. Its submitted $2,175.90 also contradicts its own repeated FICA calculation of $2,218.50."
+us,scenario_032,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,The model stopped at federal FICA and incorrectly stated that Minnesota has no mandatory employee state payroll tax. It omitted the $127.60 Minnesota paid-leave contribution.
+us,scenario_032,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model correctly calculated federal Social Security and Medicare taxes but incorrectly excluded all Minnesota employee payroll tax. The missing component is the $127.60 Minnesota paid-leave contribution.
+us,scenario_032,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model treated federal FICA as the complete payroll-tax liability and denied the applicable Minnesota employee tax. It omitted the $127.60 paid-leave contribution on the head's wages.
+us,scenario_032,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly claimed Minnesota imposes no separate employee payroll tax in 2026. Minnesota's paid-leave contribution adds $127.60 to the correctly calculated $2,218.50 of federal FICA."
+us,scenario_032,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly excluded Minnesota's 2026 paid-leave employee contribution. The contribution is $127.60 on $29,000 of Minnesota paid-leave taxable wages."
+us,scenario_032,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,"The model recognized the Minnesota paid-leave tax but applied a 0.35% employee share, producing $101.50. The applicable contribution is $127.60 on $29,000, so it understated the state component by $26.10."
+us,scenario_032,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,The model computed only the 7.65% federal FICA charge. It omitted the mandatory $127.60 Minnesota paid-leave employee contribution included in payroll tax.
+us,scenario_032,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model included only Social Security and Medicare and rounded their $2,218.50 total to $2,219. It omitted the $127.60 Minnesota paid-leave employee contribution."
 us,scenario_032,payroll_tax,gemini-3.1-pro-preview,llm_error,state_local_rule,False,The model treated 7.65% federal FICA as the entire employee payroll tax. It failed to add Minnesota's $127.60 paid-leave contribution.
-us,scenario_032,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model stopped after calculating federal FICA of $2,218.50. Minnesota's mandatory paid-leave employee contribution adds $127.60."
-us,scenario_032,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,The model equated employee payroll tax with 7.65% federal FICA alone. It omitted the $127.60 Minnesota paid-leave contribution required on the head's wages.
-us,scenario_032,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,The model included only Social Security and Medicare taxes. It omitted Minnesota's $127.60 paid-leave employee contribution from the payroll-tax total.
-us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required output contract."
-us,scenario_032,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $4,428.60 does not follow from the model's stated 6.2% Social Security and 1.45% Medicare rates on $29,000, which produce $2,218.50. It also explicitly excluded the required $127.60 Minnesota paid-leave contribution instead of summing all three employee payroll-tax components."
-us,scenario_032,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model discarded its correct $2,218.50 federal FICA calculation and invented a 5.24% effective rate based on a nonexistent Social Security wage-base interaction at $29,000. It also omitted Minnesota's $127.60 paid-leave contribution."
-us,scenario_032,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model incorrectly concluded that no state payroll tax applied and stopped at federal FICA. Minnesota's paid-leave contribution adds $127.60 to the $2,218.50 federal amount."
-us,scenario_032,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model calculated and rounded only the 7.65% federal FICA liability. It omitted Minnesota's mandatory $127.60 paid-leave employee contribution.
-us,scenario_032,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model included Social Security and Medicare taxes but omitted the $127.60 Minnesota paid-leave employee contribution. Federal FICA alone does not exhaust the requested payroll-tax categories.
-us,scenario_032,payroll_tax,grok-4.5,llm_error,state_local_rule,False,The model incorrectly stated that no mandatory Minnesota employee payroll tax applies. It therefore omitted the $127.60 paid-leave contribution from the total.
-us,scenario_032,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model stopped after summing $1,798 of Social Security tax and $420.50 of Medicare tax. It failed to include Minnesota's $127.60 paid-leave employee contribution."
-us,scenario_032,payroll_tax,inkling,llm_error,state_local_rule,False,The model incorrectly asserted that no state employee payroll taxes apply. Minnesota's paid-leave program contributes an additional $127.60 to this household's payroll tax.
-us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required output contract."
-us,scenario_032,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model correctly computed federal FICA as $2,218.50 but rounded it to $2,219 and treated it as the full liability. It omitted Minnesota's separate $127.60 paid-leave employee contribution."
-us,scenario_032,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly concluded that Minnesota has no mandatory employee payroll tax and rounded federal FICA to $2,219. The calculation must also include the $127.60 Minnesota paid-leave contribution."
-us,scenario_032,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model omitted Minnesota's $127.60 paid-leave employee contribution and misstated Medicare tax as $419.50 instead of $420.50. Its submitted $2,218.50 nevertheless corresponds to federal FICA alone, not the required three-component payroll-tax sum."
-us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the federal 185% FPL reduced-price ceiling without applying Minnesota's universal free school meals program. That state pathway assigns the household to the superior FREE tier, so reduced-price support is zero."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model classified the household using only the federal 130%-to-185% FPL bands and concluded that income above 130% barred free meals. Minnesota provides universal free meals regardless of income, which assigns the FREE tier and supersedes reduced-price eligibility."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used the federal income band as the complete eligibility test and placed the child in the reduced-price tier. It omitted Minnesota's universal free-meals pathway, under which the household receives the FREE tier and therefore no reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated an income ratio between 130% and 185% FPL as dispositive. Minnesota's universal free school meals program instead places the household in the FREE tier, which eliminates reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model repeatedly applied the ordinary federal income bands and asserted that income above 130% FPL prevented free-meal eligibility. It omitted Minnesota's universal free school meals program, which grants the FREE tier regardless of income and supersedes reduced-price meals."
-us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model stopped after comparing income with the 185% FPL ceiling. It failed to apply Minnesota's universal free school meals program, which assigns the superior FREE tier and leaves no positive reduced-price benefit."
-us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model relied exclusively on the federal 130%-to-185% FPL reduced-price range. Minnesota's universal free-meals rule places this household in the FREE tier regardless of that ratio, so reduced-price eligibility is false."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the federal reduced-price and free-meal income limits as the only tiering rules. It omitted Minnesota's universal free school meals program, which produces positive free-meal support and supersedes reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model inferred positive reduced-price support solely from income below the federal ceiling. Minnesota's universal free-meals pathway assigns the FREE tier first, so the reduced-price output remains zero."
-us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model equated membership in the ordinary federal reduced-price income range with a positive reduced-price benefit. It omitted the Minnesota universal-free classification, which supersedes that tier and yields no reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,categorical_eligibility,False,"The model correctly identified the 1.56 poverty ratio but incorrectly treated the 185% FPL test as controlling. Minnesota's universal free school meals program assigns the FREE tier regardless of income, preventing positive reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model stopped at the finding that the household's income ratio was below 185% FPL. It failed to apply Minnesota's universal free school meals program, under which the household receives the superior FREE tier rather than reduced-price meals."
-us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,categorical_eligibility,False,"The model classified the household from the federal 130%-to-185% FPL band alone. Minnesota's universal free-meals rule places the household in the FREE tier, which supersedes and zeros out reduced-price support."
-us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested output, so it failed the required structured-output contract."
-us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model used the ordinary federal income bands and school-age status but omitted Minnesota's universal free school meals program. The state rule assigns the FREE tier at the household's 1.56 poverty ratio, so reduced-price support is not positive."
+us,scenario_032,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model calculated federal FICA alone and omitted the state component. Minnesota's mandatory employee paid-leave contribution is $127.60 on the $29,000 wage base."
+us,scenario_032,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,The model applied only the combined 7.65% Social Security and Medicare rate. It omitted the $127.60 Minnesota paid-leave employee tax.
+us,scenario_032,payroll_tax,gemini-3.7-flash,llm_error,state_local_rule,False,"The model equated employee payroll tax with federal FICA and stopped at $2,218.50. It omitted Minnesota's mandatory $127.60 paid-leave contribution."
+us,scenario_032,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so the required value was missing."
+us,scenario_032,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model's stated inputs produce $1,798 of Social Security tax plus $420.50 of Medicare tax, not its submitted $4,428.60. It also incorrectly excluded the $127.60 Minnesota paid-leave contribution, so the required total is $2,346.10."
+us,scenario_032,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a 5.24% effective payroll-tax rate and an unspecified Social Security wage-base interaction even though all $29,000 of wages are subject to both Social Security and Medicare taxes. It also omitted the $127.60 Minnesota paid-leave contribution."
+us,scenario_032,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,The model correctly calculated federal FICA but incorrectly stated that no state payroll tax applied. It omitted Minnesota's $127.60 employee paid-leave contribution.
+us,scenario_032,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model rounded federal FICA to $2,219 and treated it as the full liability. It omitted the $127.60 Minnesota paid-leave contribution."
+us,scenario_032,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model included only employee Social Security and Medicare taxes. It failed to include Minnesota's mandatory $127.60 paid-leave employee contribution.
+us,scenario_032,payroll_tax,grok-4.5,llm_error,state_local_rule,False,The model incorrectly concluded that no mandatory Minnesota employee payroll tax applied. Minnesota's paid-leave contribution adds $127.60 to federal FICA.
+us,scenario_032,payroll_tax,grok-4.6,llm_error,state_local_rule,False,The model correctly computed OASDI and Medicare but incorrectly denied the Minnesota employee-side payroll tax. It omitted the $127.60 paid-leave contribution.
+us,scenario_032,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model stopped after calculating federal Social Security and Medicare taxes. It omitted the mandatory $127.60 Minnesota paid-leave contribution on the head's wages.
+us,scenario_032,payroll_tax,inkling,llm_error,state_local_rule,False,The model incorrectly stated that no state employee payroll tax applied. The missing Minnesota paid-leave contribution is $127.60.
+us,scenario_032,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so the required value was missing."
+us,scenario_032,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model calculated and rounded federal FICA but did not apply Minnesota's employee paid-leave tax. The omitted state contribution is $127.60.
+us,scenario_032,payroll_tax,ox-alpha,llm_error,state_local_rule,False,The model incorrectly asserted that Minnesota levies no mandatory employee payroll tax. It omitted the $127.60 Minnesota paid-leave contribution.
+us,scenario_032,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly concluded that Minnesota has no mandatory employee state payroll tax and rounded federal FICA to $2,219. It omitted the $127.60 Minnesota paid-leave contribution."
+us,scenario_032,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model omitted Minnesota's $127.60 paid-leave employee contribution and treated federal FICA as the full payroll tax. Its explanation also misstated Medicare tax as $419.50 even though 1.45% of $29,000 is $420.50."
+us,scenario_032,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the federal 185% FPL reduced-price ceiling as the final eligibility test and omitted Minnesota's universal free school meals pathway. That pathway assigns the superior FREE tier, which makes reduced-price support zero."
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model classified the household solely from the federal 130%-to-185% FPL band and omitted Minnesota's universal free school meals program. Universal free meals assign the FREE tier and supersede reduced-price eligibility.
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated income between 130% and 185% FPL as sufficient for positive reduced-price support. It failed to apply Minnesota's universal free meals rule, under which the child receives the superior FREE tier instead."
+us,scenario_032,reduced_price_school_meals_eligible,claude-opus-5,llm_error,categorical_eligibility,False,"The model stopped after placing household income between the federal free and reduced-price thresholds. Minnesota's universal free school meals program assigns the FREE tier regardless of that income band, eliminating reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model repeatedly applied the federal 130%-to-185% FPL classification and explicitly concluded that income above 130% prevented free meals. It omitted Minnesota's universal free school meals program, which places the household in the FREE tier and supersedes reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated income below 185% FPL as sufficient for reduced-price eligibility. It omitted Minnesota's universal free meals pathway, which assigns the FREE tier and leaves no positive reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,glm-5.2,llm_error,categorical_eligibility,False,The model used the federal 130%-to-185% FPL band as the final tier determination. Minnesota's universal free school meals rule instead assigns the household to the superior FREE tier.
+us,scenario_032,reduced_price_school_meals_eligible,gpt-5.5,llm_error,categorical_eligibility,False,"The model concluded that being above the federal free-meal limit and below the reduced-price limit produced reduced-price support. It failed to apply Minnesota's universal free school meals program, which provides free support and supersedes the reduced-price tier."
+us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model used only the federal reduced-price income ceiling and incorrectly inferred positive reduced-price support. Minnesota's universal free meals program places the household in the FREE tier, so reduced-price support is zero."
+us,scenario_032,reduced_price_school_meals_eligible,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated membership in the federal reduced-price income range as dispositive. It omitted the state universal-free pathway that assigns the better FREE tier and precludes positive reduced-price support.
+us,scenario_032,reduced_price_school_meals_eligible,grok-4.5,llm_error,categorical_eligibility,False,The model correctly identified the 1.56 poverty ratio but stopped at the 185% federal ceiling. Minnesota's universal free school meals program overrides that reduced-price classification by assigning the FREE tier.
+us,scenario_032,reduced_price_school_meals_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model assumed that a poverty ratio between 130% and 185% causes PolicyEngine to return reduced-price support. It omitted Minnesota's universal free meals rule, under which PolicyEngine returns the FREE tier instead."
+us,scenario_032,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied the 185% FPL cutoff without checking the state's universal-free pathway. Minnesota assigns free meals to all students, so the household receives FREE rather than reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,inkling,llm_error,categorical_eligibility,False,"The model classified the household from the federal 130%-to-185% FPL interval alone. Minnesota's universal free school meals program places it in the FREE tier, which supersedes reduced-price eligibility."
+us,scenario_032,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for the requested variable, so the required structured output was missing."
+us,scenario_032,reduced_price_school_meals_eligible,kimi-k3,llm_error,categorical_eligibility,False,"The model used the federal income band and the child's school age to conclude reduced-price eligibility. It omitted Minnesota's universal free meals program, which assigns the FREE tier regardless of income and eliminates reduced-price support."
+us,scenario_032,reduced_price_school_meals_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model treated income at or below 185% FPL as sufficient for reduced-price support. It failed to apply Minnesota's universal free school meals rule, which gives the child the superior FREE tier instead."
 us,scenario_032,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_032,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly used Minnesota broad-based categorical eligibility to bypass the failed SNAP gross-income screen, then contradicted its own calculation showing that net income produced a negative allotment. It replaced that result with an unsupported $493 monthly benefit despite no listed shelter expense or other deduction capable of generating it."
 us,scenario_032,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Minnesota broad-based categorical eligibility as sufficient to pass the household through the gross-income screen. Its own max-allotment-minus-30%-of-net-income calculation was negative, but it then invented a $476 monthly benefit; SNAP's minimum benefit does not convert a negative allotment for this ineligible three-person household into that amount."
@@ -2183,66 +2298,70 @@ us,scenario_032,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite
 us,scenario_032,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model stopped after subtracting an approximate joint standard deduction and taxed the remaining $217. It omitted the applicable exemption subtraction, which eliminates that residual taxable income before the 5.35% rate is applied."
 us,scenario_032,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied neither the required numeric value nor an explanation for state_income_tax_before_refundable_credits. This is a missing-output contract failure.
 us,scenario_032,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated all $13,664 of Social Security dependent benefits as Minnesota taxable income, overstating the state income base. It also failed to carry the applicable joint-filer deductions and exemptions through to zero taxable income before applying Minnesota's 5.35% rate."
-us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model approximated separate child and Working Family Credit components and asserted that Social Security income left “most” of the credit without applying the 2026 combined CWFC formula. Using $30,916 AGI and $29,000 adjusted earnings in that formula yields $2,187.60, not $2,337."
-us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Minnesota has no refundable individual income tax credits. Minnesota's refundable CWFC applies because the filer has $29,000 of adjusted earnings and one eligible child, producing $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an invented approximate $410 Working Family Credit and treated the $1,750 child amount as fully additive. The 2026 combined CWFC calculation at $30,916 AGI and $29,000 adjusted earnings produces $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model counted only the child-credit component and omitted the Working Families component despite $29,000 of filer adjusted earnings. Both components are incorporated in Minnesota's CWFC calculation, yielding $2,187.60."
-us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model approximated the shared CWFC phaseout and stopped $2.60 below the formula result. Applying the exact 2026 parameters to $30,916 AGI, $29,000 adjusted earnings, and one eligible child yields $2,187.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly put 85% of the $13,664 dependent Social Security benefits into AGI and then estimated Minnesota's credit as 35% of a federal EITC. The trace uses $30,916 AGI and Minnesota's own combined CWFC formula, which yields $2,187.60."
-us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model wrongly treated rent, education expenses, or other verified expenditures as necessary for every Minnesota refundable credit. One eligible child and $29,000 of adjusted earnings establish the CWFC pathway without those expenses, producing $2,187.60."
-us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model improperly calculated the Working Family Credit as 33% of federal EITC and added a separately inflation-adjusted $1,912 child credit. Minnesota's 2026 combined CWFC formula at the traced income inputs yields $2,187.60 rather than $3,393."
-us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model applied a child-credit phaseout starting at $35,000 even though the traced AGI is $30,916, then inserted an estimated $550 Working Family Credit. The exact combined CWFC calculation yields $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits Minnesota's CWFC eligibility pathway. The household has one eligible child and $29,000 of adjusted earnings, so the state formula generates $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model invented a flat $700 Working Family Credit and assumed no relevant reduction. Minnesota's combined CWFC formula uses both $30,916 AGI and $29,000 adjusted earnings and produces $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model inserted an approximate $1,200 Working Family Credit rather than calculating it under the 2026 CWFC parameters. The combined credit for the traced income and one child is $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model counted only a $1,750 child credit and omitted the earnings-based Working Families portion. With $29,000 of adjusted earnings, the combined CWFC amount is $2,187.60."
-us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model added an unstated $1,085 Working Family Credit to a $1,750 child amount without applying the combined statutory calculation. The 2026 CWFC formula yields $2,187.60."
-us,scenario_032,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model inflation-adjusted the child-credit maximum on its own and expressly excluded every other refundable credit. It therefore omitted the Working Families component generated by $29,000 of adjusted earnings; the combined CWFC is $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to identify Minnesota's refundable CWFC. One eligible child, $30,916 AGI, and $29,000 adjusted earnings generate $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required the prompt to supply a precomputed credit amount or additional filer details. The supplied child, income, and work facts are sufficient to calculate Minnesota's refundable CWFC as $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model assigned a flat $350 Working Family Credit and added it to $1,750. The state formula does not use that shortcut; at $30,916 AGI and $29,000 adjusted earnings, the combined amount is $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model incorrectly defined the Minnesota Working Family Credit as 25% of federal EITC. Minnesota's combined CWFC parameters applied to the traced state income measures produce $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model inserted a $1,675 Working Family Credit without applying the 2026 earnings and income rules. The correct combined calculation for one child, $30,916 AGI, and $29,000 adjusted earnings is $2,187.60."
-us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The unexplained $3,572 total implies overstated CWFC components or rates. Minnesota's formula applied to one child, $30,916 AGI, and $29,000 adjusted earnings yields $2,187.60."
-us,scenario_032,state_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model supplied a generic $500 Working Family Credit estimate and omitted the full combined child-and-work calculation. The household's traced inputs generate $2,187.60 under the CWFC."
-us,scenario_032,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly set the Working Family Credit equal to 40% of federal EITC and added an unreduced $1,750 child credit. Minnesota's combined CWFC formula instead yields $2,187.60 at the traced income levels."
-us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly used 33% of federal EITC as Minnesota's Working Family Credit and separately added $1,750. The applicable combined CWFC computation produces $2,187.60."
-us,scenario_032,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model guessed a $1,300 Working Family Credit and added it to a full $1,750 child amount. Applying the actual 2026 combined formula to $30,916 AGI and $29,000 adjusted earnings yields $2,187.60."
-us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, so the required output could not be parsed."
-us,scenario_032,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model assumed a full flat $1,250 Working Family Credit and no income reduction. The combined CWFC rules use the traced AGI and adjusted earnings and yield $2,187.60."
-us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model wrongly treated the spouse's lack of earnings and the household's income as disqualifying. The filer has $29,000 of adjusted earnings, meets the working-families requirements, and has one eligible child, generating $2,187.60."
-us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model estimated only a Working Family Credit from federal EITC percentages and omitted the child component of Minnesota's combined CWFC. The complete state calculation yields $2,187.60."
-us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $241 Child and Dependent Care Credit despite no listed care expenses and failed to calculate the eligible child's CWFC component. Minnesota's combined Child and Working Families Credits total $2,187.60."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived taxable income of about $35,708 and tax of about $3,789, then replaced that result with $3,699 on an unsupported claim about PolicyEngine-style rounding. Applying the stated bracket arithmetic yields $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $11,400 business loss as a capital loss limited to $3,000, subtracted the standard deduction while calculating AGI, and omitted the age-based and $12,000 senior deductions. A Schedule C self-employment loss offsets ordinary income in full, yielding $35,708.23 of taxable income after all deductions."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached the correct $35,708 taxable income but used an estimated $24,150 bracket cutoff and then discarded its own $3,802 calculation for an unsupported $3,416 result. The 2026 married-joint 10% cutoff is $24,800, producing $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $243 state-tax refund and used understated 2026 standard and age additions, although its intermediate estimate still approached $3,798. It then submitted $4,575 without any computation supporting that amount; the applicable deductions total $47,500 and tax is $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's taxable-income setup was only $757 low because it included the nontaxable $243 refund and introduced another unexplained $514 discrepancy. More decisively, its displayed bracket arithmetic equals about $3,698, not $3,080; correct taxable income of $35,708.23 yields $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the temporary $6,000 senior deduction for each spouse and used obsolete estimated standard-deduction parameters. It also included the $243 refund, leaving taxable income at $50,251 instead of $35,708.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $12,000 senior deduction in its explicit taxable-income calculation, which produced tax near $5,241. It then invoked unspecified adjustments to submit $3,800; applying the senior deduction and exact 2026 age additions produces $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied 2024 parameters, included the $243 refund, and omitted the $12,000 senior deduction. Those errors inflated taxable income from $35,708.23 to $50,351."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA rules expired for 2026 and substituted a smaller standard deduction, personal exemptions, and a 15% second bracket. The applicable computation uses $47,500 of deductions and the 10% and 12% married-joint brackets."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $78,818 AGI and $46,318 taxable income do not incorporate the traced income and deduction sequence. Correctly taxing $16,320 of Social Security, deducting the business loss and HSA contribution, and applying $47,500 of deductions gives $35,708.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $6,762 implies taxable income far above $35,708.23. The model omitted substantial applicable deductions, especially the age-based additions and $12,000 senior deduction, before applying the married-joint rates."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly reverted to pre-TCJA rules, using a reduced standard deduction, personal exemptions, and reverted tax brackets. The 2026 rules provide total deductions of $47,500 and tax the resulting $35,708.23 at 10% and 12%."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used only $34,000 of deductions and omitted $13,500 of applicable age-based and senior deductions. Total deductions are $47,500, reducing taxable income to $35,708.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the full $47,500 combined standard, age-based, and senior deductions were not applied. Subtracting them from $83,208.23 of AGI yields $35,708.23, not the higher taxable base underlying $5,045."
-us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and omitted the $12,000 senior deduction, using only about $34,030 of deductions. Excluding the refund and applying $47,500 of deductions reduces taxable income from its $49,421 estimate to $35,708.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model did not provide a reproducible taxable-income or bracket calculation and understated the liability by $111.99. The traced $35,708.23 taxable income produces $3,788.99 under the exact 2026 married-joint brackets."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated Social Security as nontaxable and concluded that deductions eliminated all taxable income. At this provisional income, $16,320 of Social Security is taxable, leaving $35,708.23 after the $47,500 deductions."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with personal exemptions, only $29,900 of deductions, and a 15% second bracket. The applicable rules supply $47,500 of deductions and retain the 12% bracket for this income."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $35,951 rather than $35,708.23, a $242.77 excess consistent with including the nontaxable $243 state-tax refund. Excluding that refund and applying the exact brackets yields $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly included the $243 state-tax refund, raising taxable income from $35,708.23 to about $35,951. The refund is excluded in this computation, reducing tax to $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $3,818.12 corresponds to taxing about $35,951, which is $242.77 above the traced taxable income. The model carried the $243 state-tax refund into income instead of excluding it."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $5,523 answer implies taxable income near $50,000 and therefore omits roughly the $12,000 senior deduction in addition to mismeasuring other deductions. The full deduction amount is $47,500, leaving $35,708.23 taxable."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and omitted the $12,000 senior deduction, producing $49,421 of taxable income. Correct AGI is $83,208.23 and total deductions are $47,500."
-us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and applied only $34,196 of deductions, omitting the $12,000 senior deduction and misstating the remaining deduction amount. The correct deduction total is $47,500 against AGI of $83,208.23."
-us,scenario_033,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,225 of auto-loan interest and added an unsupported temporary $1,500 standard-deduction component. The listed vehicle facts do not establish a qualifying auto-interest deduction, and the applicable combined deductions total $47,500."
-us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. Its response therefore failed the required output contract.
-us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and improperly deducted $1,225 of auto-loan interest plus $333.33 of invented overtime-premium income. The stated $53,000 annual wages do not establish qualified overtime, and no qualifying auto-purchase facts support the interest deduction."
-us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model omitted the $11,400 self-employment loss, mishandled Social Security and AGI, omitted the $12,000 senior deduction, and then submitted a negative tax amount despite calculating roughly $10,800 to $11,040. Federal income tax before refundable credits cannot represent a refund here; the correct pre-refundable liability is $3,788.99."
-us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model identified the $11,400 self-employment loss but then omitted it from AGI, using $94,608 instead of $83,208.23, and it also omitted the $12,000 senior deduction. Those two omissions substantially inflated taxable income."
-us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the deductions and Social Security treatment eliminated taxable income. In fact, $16,320 of Social Security is taxable and $83,208.23 of AGI minus $47,500 of deductions leaves $35,708.23 taxable."
+us,scenario_032,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model approximated the credit by adding guessed child and working-family components and did not apply Minnesota’s 2026 CWFC formula to $30,916 of AGI and $29,000 of adjusted earnings. That computation produces $2,187.60, not $2,337."
+us,scenario_032,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Minnesota has no refundable individual income-tax credits. The household’s age-6 dependent and $29,000 of adjusted earnings qualify it for Minnesota’s refundable Child and Working Families Credits."
+us,scenario_032,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model used a guessed $410 Working Family Credit and treated the $1,750 child amount as fully available without applying the integrated 2026 CWFC calculation. The required income inputs yield $2,187.60."
+us,scenario_032,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model counted only a $1,750 child credit and omitted the working-families portion of Minnesota’s CWFC. Applying the full credit calculation raises the refundable total to $2,187.60."
+us,scenario_032,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model rounded an estimated shared-phaseout result instead of computing Minnesota’s 2026 CWFC from $30,916 of AGI and $29,000 of adjusted earnings. The exact calculation is $2,187.60, so its $2,185 answer loses $2.60 through approximation."
+us,scenario_032,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly valued Minnesota’s credit as 35% of an estimated federal EITC and omitted the child-credit portion of the CWFC. It also overstated AGI by treating 85% of the dependent Social Security benefits as taxable, whereas the applicable AGI is $30,916."
+us,scenario_032,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model wrongly required rent, education expenses, or other expense documentation for the relevant refundable credit. One eligible child and qualifying adjusted earnings trigger Minnesota’s CWFC without those expenses."
+us,scenario_032,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model added an inflation-estimated child credit to a Working Family Credit set at 33% of federal EITC. Minnesota’s 2026 CWFC is not calculated by that federal-EITC shortcut; its prescribed calculation at the stated incomes yields $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model invented a child-credit phaseout amount of $983.60 and paired it with an estimated $550 Working Family Credit. Applying the actual 2026 CWFC parameters to $30,916 of AGI and $29,000 of adjusted earnings yields $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Minnesota’s refundable CWFC despite the household having one eligible child and qualifying work earnings. Those facts generate $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model treated the credit as a flat $1,750 child amount plus an unsupported $700 working-family amount with no phaseout. Minnesota’s 2026 formula uses both AGI and adjusted earnings and produces $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model substituted an unsupported $1,200 estimate for the working-family portion and added it to a flat $1,750 child amount. The 2026 CWFC calculation at the household’s income produces $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model counted only the Minnesota child-credit amount and omitted the working-families component generated by $29,000 of adjusted earnings. The complete CWFC is $2,187.60."
+us,scenario_032,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model added an unstated and excessive Working Family Credit to $1,750 without applying the 2026 income calculation. The combined CWFC at $30,916 of AGI is $2,187.60."
+us,scenario_032,state_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model inflation-adjusted the old $1,750 child amount on its own and incorrectly concluded that no other refundable credit applied. It omitted the working-families component of the 2026 CWFC, whose combined value is $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to identify Minnesota’s refundable CWFC. The eligible age-6 child and $29,000 of adjusted earnings produce a $2,187.60 credit."
+us,scenario_032,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly supplied credit amount as absence of eligibility. The supplied child, earnings, and AGI facts are sufficient to calculate Minnesota’s refundable CWFC of $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model assigned an unsupported flat $350 Working Family Credit and added it to $1,750. Minnesota’s 2026 CWFC calculation using $30,916 of AGI and $29,000 of adjusted earnings yields $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model incorrectly set Minnesota’s Working Family Credit to 25% of an estimated federal EITC. The state’s 2026 CWFC uses its own income-based formula and produces $2,187.60."
+us,scenario_032,state_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model assigned a $1,675 Working Family Credit without applying Minnesota’s 2026 CWFC income and phaseout rules. This substantially overstates the $2,187.60 combined refundable credit."
+us,scenario_032,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The answer implies an excessive working-families component was added to the qualifying-child credit. Applying the 2026 CWFC formula to the household’s $30,916 AGI and $29,000 adjusted earnings limits the total to $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model supplied a generic $500 estimate without calculating the child-dependent and adjusted-earnings portions of Minnesota’s CWFC. The specified inputs yield $2,187.60."
+us,scenario_032,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model incorrectly calculated the Working Family Credit as 40% of federal EITC and assumed a separate full $1,750 child credit. Minnesota’s 2026 combined formula yields $2,187.60 rather than $3,524."
+us,scenario_032,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model added an estimated $1,560 Working Family Credit to a flat child amount without applying the 2026 CWFC calculation. Zero pre-credit tax does not determine the refundable credit’s size; the income formula yields $2,187.60."
+us,scenario_032,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model incorrectly set Minnesota’s Working Family Credit at 33% of an estimated federal EITC and added a separate full child credit. The state-specific 2026 CWFC calculation produces $2,187.60."
+us,scenario_032,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used an unsupported $1,300 estimate for the Working Family Credit and added it to a flat $1,750 child amount. The actual 2026 income-based CWFC calculation yields $2,187.60."
+us,scenario_032,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response failed the required submission contract."
+us,scenario_032,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model assumed a full $1,250 Working Family Credit and no income reduction. Minnesota’s 2026 CWFC uses $30,916 of AGI and $29,000 of adjusted earnings to produce $2,187.60."
+us,scenario_032,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly required the spouse to have earnings and incorrectly placed the household above the eligibility limits. The filer’s $29,000 of adjusted earnings and one eligible child satisfy the working-families pathway and generate $2,187.60."
+us,scenario_032,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model used a guessed maximum and phaseout to estimate a $1,650 Working Family Credit while treating the child amount as separately full. Applying Minnesota’s actual 2026 CWFC parameters yields $2,187.60."
+us,scenario_032,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model estimated the Working Family Credit from federal-EITC percentages and omitted the child component of Minnesota’s CWFC. The combined state calculation for the eligible child and stated incomes is $2,187.60."
+us,scenario_032,state_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented a $241 Child and Dependent Care Credit despite no care expenses and miscalculated the relevant working-family amount. The refundable total instead comes entirely from Minnesota’s CWFC and equals $2,187.60."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $35,708 of taxable income and $3,789 of tax, then replaced that result with $3,699 based on an unsupported claim about PolicyEngine-style rounding. Rounding cannot reduce the computed liability by $90."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the $11,400 business loss as a capital loss limited to $3,000, put the standard deduction inside its AGI calculation, and omitted the age-based and $12,000 senior deductions. A Schedule C self-employment loss offsets ordinary income in full, and the combined deductions reduce taxable income to $35,708.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached the correct $35,708 taxable income but then discarded its own bracket calculation and submitted $3,416. Applying the 2026 married-joint 10% bracket through $24,800 and 12% to the remainder yields $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model improperly included the $243 state-tax refund, understated the base standard deduction, and nevertheless calculated tax near $3,798 before replacing it with $4,575. The trace excludes the refund, uses $47,500 of total standard and senior deductions, and yields $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model's stated taxable income of about $34,951 would produce roughly $3,698 under the rates it quoted, not $3,080. It also included the $243 refund; excluding it gives AGI of $83,208.23 and taxable income of $35,708.23, whose tax is $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included the $243 state-tax refund and omitted the temporary $6,000 senior deduction for each spouse. Those senior deductions add $12,000 to the joint standard and age deductions, reducing taxable income from the model's roughly $50,251 to $35,708.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first omitted the $12,000 senior deduction and computed about $5,241, then asserted that unspecified adjustments reduced the result to $3,800. The actual senior deductions produce $35,708.23 of taxable income and exactly $3,788.99 of tax, with no miscellaneous adjustment or credit."
+us,scenario_033,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used 2024 rules, included the $243 refund, and allowed only $33,100 of standard and age deductions. For 2026 the household receives $47,500 including the two senior deductions, leaving $35,708.23 taxable income."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA provisions expired for 2026 and substituted a $16,000 joint standard deduction, personal exemptions, and a 15% bracket. Current 2026 rules instead provide $47,500 in combined standard and senior deductions and apply 10% and 12% rates to $35,708.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted explanation used an unsupported AGI of $78,818 and taxable income of $46,318. The income trace gives AGI of $83,208.23 and $47,500 of deductions, producing $35,708.23 of taxable income and $3,788.99 of tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income far above the traced $35,708.23. It failed to apply the full $47,500 combined joint standard, age-based, and senior deductions before calculating the tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset regime with a reduced standard deduction, restored personal exemptions, and reverted tax brackets. The applicable 2026 computation deducts $47,500 and taxes $35,708.23 at the 10% and 12% married-joint rates."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model allowed only $34,000 of deductions and omitted the $12,000 senior deduction for the two spouses. The full deduction is $47,500, reducing taxable income to $35,708.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model did not identify or apply the full $47,500 combined standard, age-based, and senior deductions. Starting from $83,208.23 of AGI, those deductions leave $35,708.23 taxable income rather than the larger base implicit in $5,045."
+us,scenario_033,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model included the $243 state-tax refund and omitted the $12,000 deduction available for the two seniors. Correct AGI is $83,208.23 and total deductions are $47,500, not $34,030."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave no reproducible taxable-income or bracket calculation and understated the tax by $111.99. The traced taxable income is $35,708.23, and applying the 2026 married-joint 10% and 12% brackets yields $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated Social Security as nontaxable and asserted that deductions eliminated all taxable income. At this provisional-income level, $16,320 of Social Security is taxable, and after $47,500 of deductions the household still has $35,708.23 of taxable income."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model included the $243 refund and incorrectly imposed sunset-law deductions, personal exemptions, and a 15% marginal bracket. The 2026 rules exclude that refund, allow $47,500 of combined deductions, and tax $35,708.23 at 10% and 12%."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $35,951, exactly reflecting inclusion of the $243 state-tax refund. Excluding that refund produces $35,708.23 of taxable income and $3,788.99 of tax."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model included the $243 state-tax refund in AGI, raising taxable income from $35,708.23 to about $35,951. The refund is excluded in this computation, so the 2026 brackets produce $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $3,818.12 corresponds to taxing approximately $35,951 rather than the traced $35,708.23. The model implicitly included the $243 state-tax refund that must be excluded from AGI."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The answer implies omission of the two $6,000 senior deductions and therefore taxes a base near $49,000 instead of $35,708.23. The household receives $47,500 in total standard, age-based, and senior deductions."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and allowed only $34,030 of deductions, omitting the $12,000 senior deduction. Correct AGI is $83,208.23 and the full $47,500 deduction leaves $35,708.23 taxable."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied a sunset-law joint standard deduction, restored personal exemptions, and a 15% marginal rate while also including the $243 refund. The operative 2026 rules provide $47,500 of combined deductions and retain the 10% and 12% rates for this income."
+us,scenario_033,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included the $243 refund and omitted the $12,000 senior deduction, using only $34,196 of deductions. The correct inputs are $83,208.23 of AGI and $47,500 of deductions."
+us,scenario_033,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model invented a $1,225 auto-loan-interest deduction and an extra temporary $1,500 standard-deduction component. The auto-loan balance and interest do not establish the qualifying vehicle conditions, and the traced deduction total is $47,500, leaving $35,708.23 taxable."
+us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_033,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,225 of auto-loan interest and invented a $333.33 qualified-overtime deduction from wages that equal straight-time annual pay. Neither deduction applies from the listed facts; the traced taxable income is $35,708.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model omitted the $11,400 self-employment loss, mishandled Social Security and the HSA deduction, omitted the senior deductions, and then submitted a negative refund despite calculating positive tax near $11,000. Federal income tax before refundable credits cannot represent a refund, and the correct positive liability is $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model included the $243 state-tax refund, producing AGI of $83,451 and taxable income of $35,951. Excluding that refund gives AGI of $83,208.23, taxable income of $35,708.23, and tax of $3,788.99."
+us,scenario_033,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $11,400 self-employment loss when forming AGI and omitted the $12,000 senior deduction. The loss offsets ordinary income in full, and total deductions are $47,500, producing taxable income of $35,708.23."
+us,scenario_033,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and a Social Security subtraction eliminated taxable income. In fact, $16,320 of Social Security is taxable and subtracting $47,500 from $83,208.23 of AGI leaves $35,708.23 taxable."
 us,scenario_033,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_033,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,043.50, then added a fabricated $64.15 “rounding adjustment.” The component calculations are already exact to the cent and require no adjustment."
 us,scenario_033,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $6,043.50 and then improperly rounded it to $6,045. Payroll tax is requested as an annual amount, and no rule permits rounding the exact household total to a five-dollar increment."
@@ -2255,80 +2374,84 @@ us,scenario_033,payroll_tax,minimax-m3,llm_error,other,False,"The model correctl
 us,scenario_033,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly calculated each spouse's employee FICA tax and the $6,043.50 total, then improperly rounded it to $6,042. No payroll-tax computation or rounding rule produces that submitted value."
 us,scenario_033,payroll_tax,qwen3.8-max,llm_error,other,False,"The model identified the correct $79,000 wage base and combined 7.65% Social Security and Medicare rate but computed their product incorrectly. The product is $6,043.50, not $6,123."
 us,scenario_033,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly recognized that a self-employment loss produces no self-employment tax, then contradicted that rule by reporting an invented negative liability. A negative SSTB amount cannot be multiplied by the 15.3% rate to create a self-employment tax credit; the correct liability is floored at zero."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly derived $43,120 of taxable income and approximately $4,926 of tax, then replaced that result with $4,849 based on an unspecified bracket-indexing adjustment. The stated 2026 thresholds yield exactly $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of $16,100 and then miscomputed the bracket tax: its claimed $8,074 does not follow from either $44,220 of taxable income or the stated 12% bracket. The correct taxable income is $43,120 and the bracket tax is $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $16,600 standard deduction instead of $16,100 and then arbitrarily reduced its own calculated $4,866 result to $4,297 for bracket inflation. The applicable deduction and thresholds produce $4,926.40 without that adjustment."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly computed $43,120 of taxable income and $4,926 of tax, then changed the submitted value to $4,969 as a rounding adjustment. Rounding cannot turn the exact bracket calculation of $4,926.40 into $4,969."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly derived approximately $4,926 from the applicable standard deduction and brackets, then submitted $5,194 based on unspecified parameter estimates. Its own arithmetic yields $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a projected $15,750 standard deduction instead of the applicable $16,100 deduction, producing excessive taxable income. It also repeatedly inferred mortgage interest from the mortgage balance and assumed interest rates even though unlisted expenses must be zero; the correct standard-deduction calculation yields $43,120 of taxable income and $4,926.40 of tax."
-us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's stated calculation exactly yields $4,926.40, but it submitted $6,499. This is an unsupported final-value substitution that contradicts its own reasoning."
-us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the 2025 standard deduction of $15,000 instead of the 2026 amount of $16,100. That overstated taxable income by $1,100 and tax by $132, producing $5,067.90 instead of $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA sunset regime with an $8,300 standard deduction, a restored $5,300 personal exemption, and a 15% second bracket. The applicable 2026 rules use a $16,100 standard deduction, no personal exemption in this computation, and a 12% bracket over $12,400, yielding $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $5,122 answer does not apply the 2026 $16,100 standard deduction and current single-filer thresholds. Those parameters produce taxable income of $43,120 and tax of $4,926.40; the submitted amount tracks older deduction or bracket parameters."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly reinstated a personal exemption and pre-TCJA-style 15% bracket after assuming the TCJA expired. The applicable 2026 computation instead uses the $16,100 standard deduction and 10%/12% brackets, producing $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset standard deduction, restored personal exemption, and 15% second bracket. Using the applicable $16,100 deduction and 12% bracket above $12,400 yields $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly used an $8,300 standard deduction, a $5,300 personal exemption, and a 15% second bracket under a presumed TCJA sunset. The applicable 2026 parameters yield $43,120 of taxable income taxed at 10% and 12%, for $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly used post-TCJA-sunset parameters, including a restored personal exemption and a 15% second bracket. The applicable $16,100 standard deduction and 10%/12% rate schedule yield $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted 2024's $14,600 standard deduction and 2024 brackets for the requested 2026 parameters. The 2026 $16,100 deduction and $12,400 first-bracket ceiling produce $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $3,344 does not result from applying the 2026 standard deduction and ordinary-income brackets to the pension. The correct sequence is $59,220 minus $16,100 equals $43,120 of taxable income, followed by $1,240 plus $3,686.40 of bracket tax."
-us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $3,720 reflects an unsupported shortcut rather than the applicable 2026 single-filer calculation. Taxable income is $43,120 after the $16,100 standard deduction, and the 10%/12% brackets produce $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model improperly treated the mortgage balance as establishing a mortgage-interest deduction, even though no interest expense was listed and unlisted numeric inputs are zero. The taxpayer instead takes the $16,100 standard deduction, resulting in $4,926.40 of tax."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA sunset rules, using an $8,300 standard deduction, a restored $5,300 personal exemption, and a 15% second bracket. The applicable 2026 $16,100 deduction and 10%/12% brackets yield $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied invented pre-TCJA-style 2026 parameters, including a personal exemption and 15% second bracket. The applicable calculation uses a $16,100 standard deduction, no personal exemption in taxable income, and a 12% second bracket, yielding $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model deliberately substituted the 2025 $15,000 standard deduction and 2025 brackets for the requested 2026 rules. Using the 2026 $16,100 deduction lowers taxable income to $43,120 and tax to $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's stated bracket computation produced approximately $4,973, but it then submitted $6,437 without any supporting calculation. It also used an estimated $15,750 deduction instead of $16,100; the applicable parameters yield $4,926.40."
-us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model misapplied the rate schedule by placing taxable income near a 22% threshold instead of taxing $43,120 under the 10% and 12% brackets. It then invented an elderly credit for a 56-year-old and added a medical-expense deduction to tax liability rather than subtracting an allowable deduction from income; no such credit or added tax applies, so the result is $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $4,926 using the $16,100 deduction and 2026 brackets, then replaced that result with an unsupported $4,849 estimate. No further bracket indexing or rounding adjustment applies after using the governing parameters."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of $16,100 and then miscomputed the bracket tax: its stated $3,809 plus the first-bracket tax does not equal $8,074. Taxable income is $43,120, and the 10% and 12% brackets yield $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model used an estimated $16,600 deduction, derived about $4,866, and then reduced it to $4,297 without a tax rule or credit supporting that reduction. The applicable $16,100 deduction and rate schedule produce $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly computed $4,926 from taxable income of $43,120, then changed the result to $4,969 as a purported rounding adjustment. Rounding cannot alter the exact bracket calculation by $42.60."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly calculated approximately $4,926 and then submitted $5,194 based on unspecified bracket-parameter estimates. Its own stated $1,240 plus $3,686.40 equals the required $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model substituted projected parameters—a $15,750 standard deduction and an $11,950 first-bracket ceiling—for the applicable $16,100 deduction and $12,400 ceiling. It also speculated about mortgage interest from the loan balance even though no interest expense was listed; the standard-deduction calculation yields $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model explicitly performed the complete correct calculation and obtained $4,926.40, but submitted $6,499. The final value contradicts every component of its stated reasoning."
+us,scenario_036,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied the 2025 $15,000 standard deduction and 2025 $11,925 bracket threshold instead of the 2026 values. Using the $16,100 deduction and $12,400 threshold reduces taxable income to $43,120 and tax to $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset regime, replacing the $16,100 standard deduction with an $8,300 deduction plus personal exemption and replacing the 12% marginal rate with 15%. The governing 2026 parameters retain the larger standard deduction and 12% bracket, producing $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $5,122 is inconsistent with the governing 2026 deduction and brackets and aligns with using older single-filer parameters. Deducting $16,100 and applying 10% through $12,400 and 12% thereafter yields $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly imposed an expected TCJA sunset, using a small standard deduction, a restored personal exemption, and a 15% bracket. The applicable calculation uses a $16,100 standard deduction and a 12% second bracket, yielding $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied post-TCJA-sunset assumptions—an $8,300 standard deduction, a $5,050 personal exemption, and a 15% marginal rate. The applicable 2026 schedule instead uses the $16,100 standard deduction and 12% rate over $12,400, producing $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly used an $8,300 standard deduction, a restored $5,300 personal exemption, and a 15% second bracket. Using the governing $16,100 deduction and 10%/12% schedule gives $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly treated 2026 as governed by post-TCJA-sunset parameters, including a personal exemption and a 15% marginal bracket. The calculation instead deducts $16,100 and taxes $43,120 under the 10% and 12% brackets, producing $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the 2024 standard deduction and bracket thresholds as estimates for 2026. The 2026 $16,100 deduction and $12,400 first-bracket ceiling produce taxable income of $43,120 and tax of $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $3,344 does not result from applying the 2026 single standard deduction and ordinary-income brackets to the pension. Those steps produce taxable income of $43,120 and tax of $4,926.40, with no credit reducing it."
+us,scenario_036,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model asserted $3,720 without specifying the deduction, bracket thresholds, or a credit that produces that figure. The $16,100 standard deduction and 2026 10%/12% rate calculation produce $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated the mortgage balance as supporting a mortgage-interest deduction, but a balance is not deductible interest and unlisted interest expense is zero. The taxpayer instead receives the $16,100 standard deduction, leading to $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly used TCJA-sunset parameters: an $8,300 standard deduction, a $5,300 personal exemption, and a 15% second bracket. The applicable $16,100 deduction and 12% second bracket produce $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied a projected post-sunset standard deduction, personal exemption, and 15% marginal rate. The governing calculation uses a $16,100 standard deduction and the 10%/12% single schedule, yielding $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used pre-TCJA-style rules with a small standard deduction, personal exemption, and 15% second bracket. Applying the governing $16,100 deduction and 12% second bracket yields $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model deliberately used the 2025 $15,000 standard deduction and 2025 bracket schedule as a proxy for 2026. The actual 2026 $16,100 deduction and $12,400 first-bracket ceiling yield $4,926.40."
+us,scenario_036,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model first used an incorrect $15,750 deduction and derived about $4,973, then changed the output to $6,437 without any supporting rate, tax, or credit calculation. The governing parameters produce $4,926.40, and no rounding or threshold adjustment can create the submitted figure."
+us,scenario_036,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model misapplied the rate schedule, invented a $3,497.60 elderly credit for a 56-year-old, and added $6,995.20 of medical expense as tax instead of treating allowable medical expenses as a potential itemized deduction subject to the AGI floor. With the standard deduction and no nonrefundable credits, the ordinary-income tax is $4,926.40."
 us,scenario_036,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly declared the $59,220 pension income below a New Jersey adult Medicaid threshold without converting the household's MAGI to its income-to-FPL level or identifying a qualifying category. The correct computation places MAGI at 3.71 times FPL and assigns medicaid_category NONE, so the head is not eligible."
 us,scenario_036,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no wages were provided and that payroll tax therefore equals zero, but submitted $3,715 instead. That amount applies a payroll-tax-style rate to the $59,220 taxable private pension, which is not employee compensation subject to payroll tax."
 us,scenario_036,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model calculated approximately $1,674 after incorrectly deducting medical expenses, then submitted $187, a value unsupported by its own computation. It also treated $2,100 of listed premiums and expenses as generating a New Jersey medical deduction, whereas this computation subtracts only the $1,000 personal exemption and taxes $58,220."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model imported a nonexistent New Jersey standard deduction and then abandoned its own $1,417.50 bracket calculation for an unexplained $3,185 estimate. New Jersey taxable income here is $58,220 after the $1,000 personal exemption, producing $1,724.16 under the progressive schedule."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model first derived the correct $58,220 taxable income and bracket tax, then incorrectly deducted $8,405 of listed insurance premiums and medical expenses. The applicable computation subtracts only the $1,000 personal exemption from the $59,220 pension."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $8,405 of premiums and medical expenses as deductible and then invented an additional PolicyEngine ESI-premium reduction to reach $613. PolicyEngine taxes $58,220 after only the $1,000 exemption, with no such ESI adjustment."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $8,404.60 of insurance premiums and medical expenses from New Jersey income. The taxable base is $59,220 minus only the $1,000 personal exemption, so the 5.525% bracket applies to $18,220 rather than $9,815.40."
-us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model ignored the age-62 eligibility condition for New Jersey's pension exclusion and treated the income thresholds as independently sufficient. At age 56 and without disability, the filer receives no pension exclusion, so the pension remains fully included."
-us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied New Jersey's pension exclusion solely because income was below its limit, omitting the requirement that the taxpayer be at least 62 or otherwise qualifying. The 56-year-old filer's entire $59,220 pension is included."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly denied the pension exclusion but incorrectly deducted $8,404.60 of listed medical costs and premiums. Only the $1,000 personal exemption reduces income here, leaving $58,220 taxable."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The unexplained $1,113 estimate does not result from applying New Jersey's single-filer schedule to the $58,220 taxable base. The correct bracket computation is $280 + $262.50 + $175 + 5.525% of $18,220, totaling $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated income below $100,000 as sufficient for New Jersey's pension exclusion and omitted its age requirement. Because the filer is 56 and not disabled, none of the $59,220 pension is excluded."
-us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used 2% and 2.5% for New Jersey's first two single-filer brackets instead of 1.4% and 1.75%. On $58,220 of taxable income, the correct first two bracket amounts are $280 and $262.50, yielding total tax of $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model asserted $2,860 without applying the New Jersey progressive schedule. Applying the schedule to $58,220 after the $1,000 exemption yields $1,724.16, not $2,860."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invoked unspecified standard credits to reduce the liability to about $600, although no applicable nonrefundable New Jersey credit is present. Taxing $58,220 under the state schedule produces $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted a $7,304.60 medical deduction based on listed premiums and expenses. The computation subtracts only the $1,000 personal exemption, leaving $58,220 rather than $50,915.40 taxable."
-us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model granted the New Jersey pension exclusion based only on total income and omitted the age-62 or qualifying-disability condition. The 56-year-old filer receives no exclusion, so the entire pension is taxable."
-us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The unexplained $1,125 estimate does not apply New Jersey's rate schedule to the traced taxable income. The $59,220 pension less the $1,000 exemption leaves $58,220, whose scheduled tax is $1,724.16."
-us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model both deducted $8,405 of medical expenses and replaced New Jersey's progressive brackets with a fabricated $350 plus 2.75% excess-income formula. The applicable base is $58,220 and the marginal rate from $40,000 to that amount is 5.525%."
-us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced taxable income by about $8,405 for listed medical expenses and premiums. Only the $1,000 exemption is subtracted, leaving $58,220 subject to New Jersey's brackets."
-us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable.
-us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the pension exclusion requires age 62, then disregarded that condition and granted the exclusion based on income alone. At age 56 and without disability, the filer cannot exclude the $59,220 pension."
-us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model explicitly computed the correct $1,724.16 from $58,220 of taxable income, then discarded that result and submitted an unsupported $2,283.06 withholding-style estimate. Its final value contradicts its own correct bracket calculation."
-us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $16,100 New Jersey standard deduction and then added a purported property-tax deduction to tax, reversing the function of a deduction and using an unlisted expense. New Jersey taxable income here is $58,220 after the $1,000 personal exemption only."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning actually computed approximately $1,674 after its medical deduction, but it submitted $187. The submitted value is disconnected from its stated taxable-income and bracket calculation."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly introduced a $6,500 New Jersey standard deduction even though New Jersey does not use that deduction in this calculation. It then abandoned its own $1,417.50 bracket result and submitted an unsupported $3,185 instead of taxing $58,220 after the $1,000 exemption."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly reached $58,220 and $1,724.15 before wrongly deducting $8,404.60 of listed health costs. PolicyEngine's New Jersey calculation includes no medical deduction here, so taxable income remains $58,220."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted listed medical costs and then asserted that PolicyEngine further deducts the employer-sponsored insurance premium. Neither deduction enters this trace; only the $1,000 personal exemption reduces $59,220 to $58,220."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly treated $9,589 of premiums and medical spending, less the 2% floor, as a New Jersey deduction. PolicyEngine subtracts only the $1,000 exemption, so the rate schedule applies to $58,220 rather than $49,815.40."
+us,scenario_036,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model granted the New Jersey retirement-income exclusion based solely on the income threshold and omitted its age-62 eligibility condition. At age 56 and without disability, the filer receives no pension exclusion, and the submitted $1,591 also does not follow from the model's claim that taxable income was near zero."
+us,scenario_036,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the single-filer pension exclusion without enforcing the age-62 requirement. The age-56 filer's entire $59,220 pension therefore enters New Jersey adjusted gross income."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted $8,404.60 of premiums and medical expenses after a 2% floor. Those expenses do not reduce taxable income in this computation, which remains $58,220 after the personal exemption."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,113 reflects unspecified New Jersey deductions beyond the applicable $1,000 personal exemption. With no pension exclusion or other traced deduction, taxable income is $58,220 and the bracket calculation yields $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model ignored the age-62 condition for New Jersey's pension exclusion and excluded all $59,220 based only on income. The pension is fully included for this age-56 filer."
+us,scenario_036,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used incorrect 2% and 2.5% rates for the first two New Jersey brackets. The applicable rates are 1.4% through $20,000 and 1.75% from $20,000 through $35,000, followed by 3.5% and 5.525%."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $2,860 does not result from applying New Jersey's progressive schedule to $58,220. The correct schedule calculation on that taxable income is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model subtracted unspecified “standard credits” to reduce the liability to about $600, although no applicable nonrefundable New Jersey credits are present. Taxing $58,220 under the state schedule leaves $1,724.16 before refundable credits."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted $7,304.60 of premiums and medical expenses from New Jersey income. PolicyEngine deducts only the $1,000 personal exemption, leaving $58,220 rather than $50,915.40 taxable."
+us,scenario_036,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model granted the pension exclusion from the income threshold alone and omitted the age-62 condition. Because the filer is 56, all $59,220 of pension income remains in New Jersey adjusted gross income."
+us,scenario_036,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The unexplained $1,125 implies deductions or credits not present in the New Jersey computation. Only the $1,000 exemption reduces the $59,220 pension, and the tax on $58,220 is $1,724.16."
+us,scenario_036,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model both deducted $8,405 of medical expenses that do not enter this trace and replaced New Jersey's progressive schedule with “$350 plus 2.75% of excess over $20,000.” The applicable taxable income is $58,220 and must be taxed through the 1.4%, 1.75%, 3.5%, and 5.525% brackets."
+us,scenario_036,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model improperly reduced income by approximately $8,405 for medical costs. It also misapplied the brackets to its own $49,815 taxable-income figure; PolicyEngine instead taxes $58,220 after only the personal exemption."
+us,scenario_036,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the submission contract."
+us,scenario_036,state_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the filer is under 62 but then incorrectly treated the income-based exclusion limit as an independent eligibility route. The age requirement bars the pension exclusion, so the $59,220 pension is fully included."
+us,scenario_036,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted $1,015.60 of medical expenses after applying a 2% floor. PolicyEngine uses no medical deduction here, leaving taxable income of $58,220 after the $1,000 exemption."
+us,scenario_036,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model explicitly derived the correct $58,220 taxable income and $1,724.16 tax, then discarded that result for an unsupported $2,283.06 “withholding-style” estimate. State income tax before refundable credits uses the actual progressive liability calculation, not a withholding approximation."
+us,scenario_036,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $16,100 New Jersey standard deduction and then added a purported property-tax deduction to tax, reversing the function of a deduction. No property tax was listed, and the traced calculation subtracts only the $1,000 personal exemption before applying the rate schedule."
 us,scenario_036,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model explicitly rejected the qualified overtime deduction and used an obsolete $14,600 standard deduction instead of $16,100. Its submitted $1,646 also contradicts its own displayed $2,308 computation rather than following any stated calculation."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted the retirement adjustments and standard deduction but omitted the separate $9,629.03 qualified overtime deduction. That left taxable income near $25,439 instead of $15,810.43, and its $2,884 submission also exceeds its own approximate $2,805 tax calculation."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified overtime deduction, leaving taxable income near $25,439 rather than $15,810.43. It then replaced its own calculated tax of about $2,805 with an unsupported $3,160 figure."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model recognized the qualified overtime deduction but did not actually subtract the full $9,629.03 in its tax calculation. Applying that deduction alongside the $16,100 standard deduction reduces taxable income to $15,810.43 and tax to $1,649.25, not $2,570."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model eventually applied the overtime amount but used estimated standard deductions and bracket thresholds instead of the 2026 values: a $16,100 standard deduction and the applicable 2026 rate schedule. It also submitted $1,569 despite its final displayed computation producing $1,700."
-us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified overtime deduction and therefore computed tax on about $25,439 rather than $15,810.43. Its $1,808 submission also does not follow its own displayed tax calculation of $2,809.68."
-us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the separately listed ESI premium from wages, assumed a post-TCJA reversion with a small standard deduction and personal exemption, and omitted the $9,629.03 qualified overtime deduction. The operative calculation instead uses AGI of $41,539.45 and total deductions of $25,729.03."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly treated the ESI premium as an additional reduction from stated gross wages, assumed the TCJA rules reverted, and omitted the qualified overtime deduction. The 2026 computation uses the $16,100 standard deduction plus $9,629.03 of deductible overtime, with no personal exemption."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's explanation omitted both the $18.03 above-the-line deduction and the $9,629.03 qualified overtime deduction and used an estimated $15,800 standard deduction. Its $1,109 answer also does not result from applying the 2026 brackets to the taxable income described in its explanation."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the ESI premium again, assumed a TCJA sunset with a personal exemption, and omitted the $9,629.03 qualified overtime deduction. It also invented a $50 Saver's Credit instead of deriving the liability from the traced deductions and applicable credits."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied an inapplicable TCJA-expiration regime, including a personal exemption and 15% bracket, while double-counting the ESI premium as an AGI reduction. It also omitted the $9,629.03 qualified overtime deduction that brings taxable income to $15,810.43."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $5,789 of ESI premiums from the stated wages and used a $15,450 standard deduction. More importantly, it omitted the $9,629.03 qualified overtime deduction and therefore taxed $20,300 rather than $15,810.43."
-us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's terse derivation double-counted the ESI premium as a wage reduction and failed to include the full $9,629.03 qualified overtime deduction. It also subtracted a Saver's Credit that does not affect the traced liability."
-us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $14,600 standard deduction and entirely omitted the $9,629.03 qualified overtime deduction. Those errors inflated taxable income to $26,939 instead of $15,810.43."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction and unspecified credits as sufficient to eliminate all tax. The actual deductions leave $15,810.43 of taxable income, and no nonrefundable credit reduces the resulting $1,649.25 liability to zero."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed the standard deduction and unspecified education or retirement credits fully offset the liability. After all applicable deductions, $15,810.43 remains taxable, and no such nonrefundable credits apply."
-us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $385.90 pre-tax 401(k) reduction and the $9,629.03 qualified overtime deduction, while using a $15,750 standard deduction instead of $16,100. This inflated taxable income from $15,810.43 to its stated $26,175."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's stated shortcut accounted only for retirement contributions and the standard deduction, omitting the separate $9,629.03 qualified overtime deduction. That omission leaves taxable income roughly $9,629 too high and produces tax near $2,894 instead of $1,649.25."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a post-TCJA reversion with an $8,300 standard deduction, personal exemption, and 15% bracket. It also omitted the $9,629.03 qualified overtime deduction; the operative total deduction is $25,729.03 and no personal exemption is used."
-us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted ESI premiums as an AGI reduction and applied an inapplicable reverted-tax regime with a personal exemption and 15% bracket. It omitted the $9,629.03 qualified overtime deduction and the operative $16,100 standard deduction."
-us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $5,789 ESI premium from stated wages, omitted the $9,629.03 qualified overtime deduction, and then applied an unsupported $40.40 Saver's Credit. The traced computation instead leaves taxable income of $15,810.43 with no credit changing the resulting tax."
-us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive computation."
-us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that wages of $41,943 fall below the standard deduction and personal exemption. Even after the retirement adjustments, $16,100 standard deduction, and $9,629.03 overtime deduction, taxable income remains $15,810.43."
-us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $18.03 traditional IRA deduction and the entire $9,629.03 qualified overtime deduction, while using an estimated $15,600 standard deduction instead of $16,100. That produced taxable income of $25,957 rather than $15,810.43."
-us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a Child Tax Credit for a household with no child or other dependent listed. No CTC is available, so nonrefundable credits do not erase the $1,649.25 tax produced after the applicable deductions."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model expressly rejected the qualified-overtime deduction and also treated the separately listed $5,789 employer-sponsored insurance premium as an additional reduction from gross wages. The overtime premium instead supplies a $9,629.03 deduction below AGI, producing $15,810.43 of taxable income and $1,649.25 of tax."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted the retirement adjustments and standard deduction but entirely omitted the separate $9,629.03 qualified-overtime deduction. Its own taxable-income estimate of about $25,439 is therefore $9,629 too high."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified-overtime deduction, leaving taxable income near $25,439 instead of $15,810.43. It then abandoned its own approximately $2,805 bracket calculation and submitted $3,160 without a supporting computation."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model identified the qualified-overtime deduction but did not actually subtract its full $9,629 amount: doing so reduces taxable income from about $25,439 to $15,810.43, not merely enough to yield $2,570. Applying the 2026 brackets to the correctly reduced income gives $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated qualified overtime as an exclusion from gross income rather than a deduction taken alongside the standard deduction, and it used invented standard-deduction and bracket estimates instead of the 2026 parameters. Those errors led it away from taxable income of $15,810.43 and tax of $1,649.25; its submitted $1,569 also contradicts every calculation in its explanation."
+us,scenario_037,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the entire $9,629.03 qualified-overtime deduction, calculating $25,439 of taxable income rather than $15,810.43. Its submitted $1,808 also does not follow from its stated $2,809.68 tax calculation."
+us,scenario_037,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a supposed TCJA sunset, personal exemption, reduced standard deduction, and 15% bracket instead of the operative 2026 rules, while also deducting the listed ESI premium from wages and omitting the qualified-overtime deduction. The correct deductions are the $16,100 standard deduction plus $9,629.03 for qualified overtime, yielding $15,810.43 of taxable income."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a TCJA-expiration regime with a personal exemption, smaller standard deduction, and 15% bracket, and it double-counted the ESI premium as a wage reduction. It also omitted the $9,629.03 qualified-overtime deduction that reduces taxable income to $15,810.43."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The explanation applies brackets after only a standard deduction and the 401(k) adjustment, omitting both the $18.03 above-the-line deduction and the $9,629.03 qualified-overtime deduction. Its $1,109 result also cannot follow from the taxable income implied by the stated inputs."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the $5,789 ESI premium as a reduction from stated wages, substituted an inapplicable TCJA-sunset standard deduction and personal exemption, and omitted the $9,629.03 qualified-overtime deduction. It additionally invented a $50 Saver's Credit instead of deriving tax from the correct $15,810.43 taxable income."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model imposed an inapplicable TCJA-sunset regime with a personal exemption and 15% bracket, double-counted ESI premiums as a wage reduction, and omitted the qualified-overtime deduction. Under the operative rules, the standard and overtime deductions total $25,729.03 and leave $15,810.43 taxable."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reduced wages by the separately listed ESI premium and used a $15,450 standard deduction, but never applied the $9,629.03 qualified-overtime deduction. The operative $16,100 standard deduction and overtime deduction leave $15,810.43 of taxable income."
+us,scenario_037,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model reduced wages by ESI premiums, omitted the $9,629.03 qualified-overtime deduction, and then applied a Saver's Credit. The required computation instead uses AGI of $41,539.45 and combined standard and overtime deductions of $25,729.03, producing $1,649.25 before refundable credits."
+us,scenario_037,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $9,629.03 qualified-overtime deduction and used a $14,600 standard deduction instead of $16,100. Those mistakes inflated taxable income to $26,939 rather than $15,810.43."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that the standard deduction and unspecified credits eliminate all liability. After the retirement adjustments, standard deduction, and qualified-overtime deduction, $15,810.43 remains taxable and generates $1,649.25; no nonrefundable credit reduces it to zero."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as a credit and invented unspecified education or retirement credits sufficient to erase the tax. The deductions leave $15,810.43 of taxable income, and no applicable nonrefundable credit offsets the resulting $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $385.90 pre-tax 401(k) reduction in its stated taxable-income calculation and the $9,629.03 qualified-overtime deduction. Correctly applying those adjustments and the $16,100 standard deduction produces taxable income of $15,810.43, not $26,175."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its terse derivation accounts only for retirement adjustments and the standard deduction, so the submitted $2,894 reflects omission of the $9,629.03 qualified-overtime deduction. That additional deduction lowers taxable income to $15,810.43 and tax to $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied an inapplicable sunset regime with an $8,300 standard deduction, personal exemption, and 15% bracket, while omitting the $9,629.03 qualified-overtime deduction. The operative deductions are $16,100 plus $9,629.03, leaving $15,810.43 taxable under the 2026 rate schedule."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model recognized the overtime deduction but used a $15,400 standard deduction instead of $16,100 and treated overtime as reducing AGI rather than as a below-AGI deduction. These mistakes left taxable income $700 too high and produced $1,736 instead of $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted ESI premiums as a wage reduction and used an inapplicable reverted regime with a personal exemption, smaller standard deduction, and 15% bracket. It omitted the separate $9,629.03 qualified-overtime deduction required alongside the $16,100 standard deduction."
+us,scenario_037,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model reduced stated wages by the $5,789 ESI premium, omitted the $9,629.03 qualified-overtime deduction, and applied a $40.40 Saver's Credit. The correct path starts from AGI of $41,539.45 and subtracts $25,729.03 of standard and overtime deductions, yielding $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested variable, so its response failed the required output contract."
+us,scenario_037,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed that $41,943 of wages falls below the standard deduction and personal exemption. Even after the $16,100 standard deduction and $9,629.03 qualified-overtime deduction, taxable income is $15,810.43, so liability is not zero."
+us,scenario_037,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model correctly computed AGI and the standard deduction but omitted the additional $9,629.03 qualified-overtime deduction. Subtracting it reduces taxable income from $25,439 to $15,810.43 and tax from $2,804.68 to $1,649.25."
+us,scenario_037,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $18.03 deductible traditional IRA contribution, used an estimated $15,600 standard deduction instead of $16,100, and entirely omitted the $9,629.03 qualified-overtime deduction. Those omissions inflated taxable income to $25,957 instead of $15,810.43."
+us,scenario_037,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a Child Tax Credit even though the household contains no child or other dependent. No CTC is available, so the $1,649.25 tax generated after the applicable deductions is not reduced to zero."
 us,scenario_037,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output for federal_refundable_credits and therefore failed the required submission contract.
 us,scenario_037,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented dependent-child credit eligibility despite the household containing only one 44-year-old adult: refundable CTC is $0 because there is no qualifying child. It also applied a child-based EITC amount even though the filer has no qualifying children, and $41,943 exceeds the childless EITC income limit, so EITC is $0."
 us,scenario_037,head_medicaid_eligible,glm-5.2,llm_error,thresholds_rates,False,"The model treated North Carolina’s adoption of Medicaid expansion as sufficient for eligibility and failed to compare the head’s MAGI income of 2.60 times FPL with the expansion-adult income limit. The head exceeds that limit and has no alternative Medicaid pathway, yielding medicaid_category NONE and ineligibility."
@@ -2344,31 +2467,33 @@ us,scenario_037,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
 us,scenario_037,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model treated the $5,789 employer-sponsored insurance premium as an exclusion from FICA wages, reducing the base to $36,154. The full $41,943 remains subject to employee Social Security and Medicare tax."
 us,scenario_037,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, violating the required structured-output contract."
 us,scenario_037,payroll_tax,minimax-m3,llm_error,other,False,"The model used coarse approximations for the Social Security and Medicare components and then overstated their sum as $3,213. Applying the rates and engine rounding yields $2,600.49 and $608.18, totaling $3,208.67."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used the wrong $10,750 standard deduction and then discarded its own positive tax calculation by inventing unspecified deductions or credits that reduced liability to zero. The applicable $12,750 deduction leaves $28,789.45 taxable and produces $1,148.70, with no applicable nonrefundable credit eliminating it."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the listed $41,943 gross wages as though the traditional 401(k) contribution had already been excluded, despite initially identifying that contribution as an AGI deduction. Subtracting the $386 traditional 401(k) and $18 traditional IRA contributions yields $41,539.45 of AGI and $1,148.70 of tax."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived approximately $28,789 of taxable income and approximately $1,152 using its rounded 4% rate, then arbitrarily increased the result to $1,180 for nonexistent “traditional contribution effects.” Those contributions were already reflected in AGI, and the engine calculation yields $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reached approximately $1,149 from $28,789 of taxable income at 3.99%, then imposed an unsupported $266 reduction for unspecified decoupling or deduction treatment. No such adjustment applies, so the computed liability remains $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model repeatedly reached the correct $28,789 taxable income and $1,148.68 tax, but abandoned that computation and selected a fabricated $21,500 single standard deduction to force an approximately $803 result. The applicable North Carolina single standard deduction is $12,750."
-us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly calculated $28,789 of taxable income and a tax near the reference, then replaced that result with $1,685 based on unspecified rounding and parameter updates. Neither rounding nor a standard-deduction update can turn its stated $1,223.50 calculation into $1,685; the applicable computation yields $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $35,750 instead of $41,539.45, thereby reducing North Carolina taxable income to $23,000. It improperly removed an additional $5,789 from wages; the employer-sponsored insurance premium is not separately deductible in this calculation."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 by effectively deducting the $5,789 employer-sponsored insurance premium in addition to the retirement deductions. Correct AGI is $41,539.45, and it also used $12,850 rather than the applicable $12,750 North Carolina standard deduction."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $1,787.52 is consistent with applying its approximate 4.3% rate to nearly all $41,539 of AGI without properly subtracting the $12,750 North Carolina standard deduction. The applicable taxable base is $28,789.45 and the 2026 calculation yields $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $35,750 and therefore taxed only $23,000 after the state deduction. Correct AGI is $41,539.45; the answer reflects an improper extra $5,789 deduction matching the employer-sponsored insurance premium."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $35,750 as state adjusted gross income, effectively deducting the $5,789 employer-sponsored insurance premium a second time. North Carolina starts from $41,539.45 of AGI, leaving $28,789.45 after the $12,750 standard deduction."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated starting AGI by $5,789, exactly the listed employer-sponsored insurance premium, and consequently understated taxable income as $23,000. That premium is not an additional deduction from the stated wages; taxable income is $28,789.45."
-us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $917.70 answer at 3.99% implies only $23,000 of taxable income, rather than the required $28,789.45. This shortcut reflects an extra $5,789 reduction matching the employer-sponsored insurance premium, which is not separately deductible from wages."
-us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used an invented $15,500 North Carolina single standard deduction instead of $12,750. This understated taxable income by $2,750 and reduced the tax from $1,148.70 to about $1,039."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $1,268 submission does not apply the governing North Carolina parameters to the stated income. AGI of $41,539.45 less the $12,750 standard deduction leaves $28,789.45, whose 2026 tax is $1,148.70 rather than $1,268."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and credits erased all taxable income. The $12,750 standard deduction leaves $28,789.45 taxable, and no applicable nonrefundable state credit reduces the resulting $1,148.70 liability to zero."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model correctly derived $28,789 of taxable income and $1,148.68 of tax, but submitted $1,147 instead of rounding or retaining its computed amount. The trace's unrounded inputs produce $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced North Carolina taxable income to $19,160 through unsupported additional above-the-line deductions. The listed traditional retirement deductions produce $41,539.45 of AGI, and subtracting the $12,750 state standard deduction leaves $28,789.45."
-us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model's unexplained $1,370 applies the wrong tax base or rate. The governing computation taxes $28,789.45 after the $12,750 standard deduction and yields $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750, effectively subtracting the $5,789 employer-sponsored insurance premium as an additional deduction. Correct AGI is $41,539.45 and taxable income is $28,789.45; it also rounded the 3.99% rate to 4%."
-us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an AGI of about $35,750 rather than $41,539.45, which lowered taxable income to $23,000. The $5,789 difference matches an impermissible separate deduction for the employer-sponsored insurance premium."
-us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model explicitly deducted the $5,789 employer-sponsored insurance premium from gross wages when computing federal AGI. That premium is not separately subtracted here; only the applicable traditional retirement contributions reduce AGI to $41,539.45, leaving $28,789.45 taxable in North Carolina."
-us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted that pre-tax adjustments and the standard deduction reduced taxable income to nearly zero, despite $41,539.45 of AGI and only a $12,750 North Carolina standard deduction. It also used the wrong 4.5% rate; the applicable computation yields $1,148.70."
-us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an obsolete 4.5% rate instead of the governing 2026 North Carolina rate and omitted the deductible $18 traditional IRA contribution from AGI. It then submitted $1,773.56 despite repeatedly computing about $1,296 under its own incorrect assumptions, so the final number follows neither its stated base nor its stated rate."
-us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's unexplained $1,234 estimate does not apply the governing 2026 parameters. Subtracting the $12,750 standard deduction from $41,539.45 of AGI leaves $28,789.45, producing $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $10,750 standard deduction and then discarded its own positive tax calculation by asserting that unspecified deductions or credits reduced liability to zero. With no applicable nonrefundable credits, $28,789.45 is taxed at 3.99% rather than eliminated."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the stated gross wages as already net of the traditional contributions and therefore failed to reduce AGI to $41,539.45. Subtracting the $12,750 standard deduction from the correct AGI yields $28,789.45 of taxable income and $1,148.70 of tax."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly derived approximately $28,789 of taxable income and approximately $1,152 using its rounded 4.0% rate, but then increased the result to $1,180 without a valid adjustment. Applying the exact 3.99% rate produces $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed approximately $1,149 and then replaced it with $883 based on an unspecified decoupling adjustment. No such adjustment applies here, so the computed $28,789.45 base remains subject to the 3.99% rate."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model reached the correct $28,789 taxable income and $1,148.68 calculation, then abandoned it and invented a $21,500 single standard deduction to force the answer to $803. The applicable deduction is $12,750, not $21,500."
+us,scenario_037,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an estimated 4.25% rate instead of the 2026 rate of 3.99%, then raised even its own $1,223.50 calculation to $1,685 without a tax-base or credit computation supporting the adjustment. The correct base and rate yield $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI as $35,750 instead of $41,539.45. It thereby taxed only $23,000 rather than the correct $28,789.45 after the $12,750 standard deduction."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 and also used a $12,850 standard deduction rather than $12,750. Those errors reduced taxable income to $22,900 instead of $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,787.52 is consistent with applying an approximate rate directly to too much income rather than first reducing $41,539.45 of AGI by the $12,750 standard deduction. The correct taxable base is $28,789.45 and the applicable rate is 3.99%."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly started from AGI of $35,750, leaving only $23,000 after the state standard deduction. The correct AGI is $41,539.45, leaving $28,789.45 taxable."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used state adjusted gross income of $35,750. Subtracting the $12,750 standard deduction from the correct $41,539.45 AGI gives $28,789.45, not $23,000."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated the North Carolina starting AGI as $35,750. The correct starting amount is $41,539.45, so taxable income after the $12,750 deduction is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $917.70 result implies a $23,000 taxable base, which comes from using the erroneous $35,750 AGI shared by the other answers. The correct taxable base is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented an approximately $15,500 North Carolina standard deduction. The applicable single deduction is $12,750, leaving $28,789.45 rather than $26,039 taxable."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $1,268 estimate does not apply the governing parameters to the stated income. Reducing $41,539.45 by $12,750 and applying 3.99% produces $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and credits eliminate the tax. The $12,750 standard deduction leaves $28,789.45 taxable, and no applicable nonrefundable credit reduces the resulting $1,148.70 liability to zero."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,other,False,"The model correctly derived $28,789 of taxable income and $1,148.68 of tax but submitted $1,147 instead of rounding its calculation consistently. The trace's full-precision inputs produce $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $31,910 and therefore arrived at only $19,160 of taxable income, reflecting an unsupported additional $9,629 deduction. The FLSA overtime premium is not a deduction from wages; taxable income is $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The bare $1,370 estimate does not result from applying the 2026 parameters to the household. The $28,789.45 taxable base at 3.99% yields $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used AGI of $31,910, subtracting the separately listed $9,629 overtime premium as though it were a deduction. Overtime premium is wage information, not an above-the-line deduction, so AGI remains $41,539.45 after the traditional contributions."
+us,scenario_037,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,750 and therefore taxed only $23,000. It also rounded the rate to 4%; the correct calculation uses $28,789.45 at 3.99%."
+us,scenario_037,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly used federal AGI of about $35,750, producing a $23,000 taxable base. Correct AGI of $41,539.45 less the $12,750 deduction leaves $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model deducted the $5,789 employer-sponsored insurance premium from gross wages even though PolicyEngine's AGI derivation does not treat that listed premium as an additional deduction from this wage input. This lowered AGI from $41,539.45 to $35,750 and taxable income from $28,789.45 to $23,000."
+us,scenario_037,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted that pre-tax adjustments and the standard deduction reduced taxable income to nearly zero without identifying deductions large enough to do so. The allowed adjustments leave AGI of $41,539.45 and taxable income of $28,789.45."
+us,scenario_037,state_income_tax_before_refundable_credits,ox-alpha,llm_error,other,False,"The model's explanation correctly calculated $1,148.68, but its recorded answer was $1,223.53, so the submitted value contradicted its own final computation. The full-precision trace yields $1,148.70."
+us,scenario_037,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% rate instead of 3.99%, omitted the $18 IRA reduction from its main base, and then submitted $1,773.56 despite repeatedly calculating about $1,296. The applicable calculation is $28,789.45 times 3.99%."
+us,scenario_037,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The unexplained $1,234 estimate does not match the 2026 North Carolina rate applied to the correct taxable base. Applying 3.99% to $28,789.45 produces $1,148.70."
 us,scenario_038,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model applied only CHIP's age and upper-income criteria. It omitted the prerequisite that a child must not qualify for Medicaid; this seven-year-old qualifies under Louisiana's OLDER_CHILD Medicaid category and is therefore ineligible for CHIP.
 us,scenario_038,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated income below roughly 250% FPL as sufficient for CHIP. It failed to evaluate Medicaid first, where the child qualifies under the OLDER_CHILD category, which automatically bars CHIP eligibility."
 us,scenario_038,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used the CHIP upper-income limit as a standalone eligibility test, including a self-employment-tax adjustment that does not resolve the controlling issue. The child qualifies for Medicaid under Louisiana's OLDER_CHILD pathway, so CHIP eligibility is false regardless of being below CHIP's ceiling."
@@ -2398,35 +2523,37 @@ us,scenario_038,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibili
 us,scenario_038,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly placed a six-year-old within the modeled WIC child age category. Because eligibility ends at the fifth birthday, Child 2 fails the categorical age test."
 us,scenario_038,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable output for the requested variable and therefore failed the required submission contract.
 us,scenario_038,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's reasoning correctly derived zero taxable income, zero tax before credits, and zero nonrefundable credits used, but its submitted value of -$1,644 contradicts that derivation. It inserted an unsupported negative amount even though nonrefundable credits cannot reduce liability below zero and refundable credits are excluded from this output."
-us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model replaced the 2026 two-child EITC of $7,316 with an unsupported $7,239 estimate in its submitted total. Its own refundable CTC calculation was approximately correct, so the $77.35 shortfall comes from overriding the applicable EITC parameter."
-us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model imposed a $1,656.20 EITC phaseout even though this married couple's income remains in the two-child EITC plateau, where the credit is $7,316. It also treated the refundable CTC as the full $3,400 cap instead of applying the 15% earned-income limit, which yields $2,909.35."
-us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model understated both components: the applicable two-child EITC is $7,316, not about $7,150, and its stated 15% calculation yields about $2,909 rather than the unexplained $2,383 it used. Correctly summing those components gives $10,225.35."
-us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model reduced a plateau-level two-child EITC to roughly $5,500 and then cut the refundable CTC to $1,527 despite computing an earnings-based amount near $3,074. The applicable calculations produce a $7,316 EITC and $2,909.35 refundable CTC."
-us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model stated component estimates near $7,000 and $3,074 but submitted $9,199, which does not equal those figures. It also used gross earned income for the refundable CTC instead of reducing self-employment earnings by the deductible half of self-employment tax; the correct components are $7,316 and $2,909.35."
-us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted obsolete 2025 EITC parameters and ultimately used a $6,604 maximum instead of the 2026 two-child amount of $7,316. Its refundable CTC of about $2,909 followed the correct earnings limitation, leaving the EITC parameter as the decisive error."
-us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model initially derived components near $6,900 and $2,896 but then arbitrarily reduced their sum to $8,500. It failed to apply the exact 2026 EITC amount of $7,316 and the refundable CTC calculation of $2,909.35."
-us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child refundable CTC and a $3,000 earnings threshold, limiting that component to $2,000. The 2026 computation applies the $2,500 threshold and produces $2,909.35, while the EITC is $7,316 rather than $7,354."
-us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child refundable CTC and $3,000 earnings threshold, producing only $2,000. The refundable CTC is $2,909.35 under the applicable 15% formula, and the exact EITC is $7,316."
-us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated the full $4,000 Child Tax Credit as refundable instead of applying the refundable portion's 15% earned-income limitation, which yields $2,909.35. It also reduced the EITC to $4,100 even though the household receives the $7,316 plateau amount."
-us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assigned only $2,000 to the refundable Child Tax Credit, omitting $909.35 generated by the applicable 15% earnings formula. It also rounded the EITC down from $7,316 to $7,300."
-us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child ACTC amount, limiting the refundable CTC to $2,000 rather than $2,909.35. It also understated the exact 2026 two-child EITC by $66."
-us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted total implies that the model understated the combined EITC and refundable CTC by $471.91. The required component calculation is $7,316 plus $2,909.35, not the unexplained aggregate of $9,753.44."
-us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model identified the two relevant credits but understated their sum by $40.35 without providing a component calculation. Applying the exact $7,316 EITC and $2,909.35 refundable CTC yields $10,225.35."
-us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not be parsed or substantively evaluated."
-us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model named the EITC and refundable CTC but submitted an unexplained total $1,501.35 below their applicable sum. The exact components are $7,316 and $2,909.35."
-us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that low earnings prevent refundable credits. Two qualifying children and the household's earned income generate a $7,316 EITC and $2,909.35 refundable CTC even with zero regular income tax."
-us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly calculated the $7,316 EITC but treated $1,800 per child as automatically refundable. The refundable CTC is constrained by the 15% earned-income formula to $2,909.35, not the $3,600 cap."
-us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an $8,231 EITC maximum instead of the applicable two-child amount of $7,316. It also used gross income in the refundable CTC formula, producing $3,073.95 instead of $2,909.35 after the self-employment-tax deduction."
-us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model correctly used the $7,316 EITC but understated the refundable CTC as $2,731.32. Applying 15% to qualifying earned income above $2,500 yields $2,909.35."
-us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model confused zero pre-credit income tax with zero refundable credits. The listed earned income and two qualifying children directly establish a $7,316 EITC and a $2,909.35 refundable CTC; no unlisted facts are required."
-us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child refundable CTC ceiling, limiting the credit to $2,000. The earnings formula produces $2,909.35 under the applicable refundable cap, and the exact EITC is $7,316 rather than $7,312."
-us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied post-TCJA-reversion assumptions—$1,000 per child and a $3,000 threshold—that do not govern this 2026 calculation. The refundable CTC is $2,909.35, and the exact two-child EITC is $7,316 rather than $7,338."
-us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model overstated the EITC as about $7,422 instead of $7,316 and understated the refundable CTC as about $2,879 instead of $2,909.35. Those parameter and earnings-base errors produced the $75.65 overstatement."
-us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not be parsed or substantively evaluated."
-us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $7,316 EITC but applied the refundable CTC formula to gross wages plus gross self-employment income. Self-employment earnings must be reduced by the deductible half of self-employment tax, producing refundable CTC of $2,909.35 rather than $3,073.95."
-us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated the $3,400 per-child refundability ceiling as the refundable CTC amount instead of applying the lower 15% earned-income limit of $2,909.35. It also used an approximate $6,500 EITC rather than the applicable $7,316."
-us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"After deriving totals near the correct range, the model discarded its work and submitted $6,868.75 with no refundable CTC included. It also used obsolete EITC maxima and phaseout thresholds; the applicable components are a $7,316 EITC and $2,909.35 refundable CTC."
-us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated $22,993 as almost fully phasing out the two-child EITC, reducing it to $46. For married filing jointly this income remains in the 2026 plateau and yields $7,316; the refundable CTC is $2,909.35 rather than the rounded $3,000."
+us,scenario_038,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived refundable CTC near $2,909 but replaced the 2026 two-child EITC maximum of $7,316 with an unsupported $7,239 estimate. Its own preceding calculations produced totals above $10,300, so the submitted $10,148 also contradicts its stated computation."
+us,scenario_038,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model used the wrong EITC phase-in endpoint and then imposed a $1,656.20 phaseout even though this married couple's income remains below the joint-filer phaseout threshold. It also awarded the full $3,400 refundable CTC cap instead of applying the earned-income limit, which yields $2,909.35."
+us,scenario_038,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model understated both components: the applicable two-child EITC plateau amount is $7,316, not its approximate $7,150, and its own 15% calculation gives refundable CTC of about $2,909 rather than $2,383. The submitted total therefore does not follow its stated arithmetic."
+us,scenario_038,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model reduced the plateau EITC to roughly $5,500 despite recognizing that the household is near the plateau; the correct plateau credit is $7,316. It also cut refundable CTC to $1,527 even though the 15% earned-income formula yields $2,909.35."
+us,scenario_038,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model identified an EITC near $7,000 and refundable CTC near $3,074, but submitted $9,199, which is not their sum. It also used gross self-employment income in the CTC phase-in instead of earned income after the deductible half of self-employment tax, and failed to apply the exact $7,316 EITC maximum."
+us,scenario_038,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used obsolete 2025 EITC parameters and set the plateau credit to $6,604 rather than the 2026 amount of $7,316. Its final refundable CTC calculation was essentially correct at $2,909, so the EITC parameter error accounts for the shortfall."
+us,scenario_038,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model's own component estimates of about $6,900 EITC and $2,896 refundable CTC total about $9,796, yet it arbitrarily adjusted the result to $8,500. It also treated the 92.35% self-employment-tax base as net EITC earnings instead of subtracting half of self-employment tax and failed to use the exact $7,316 plateau credit."
+us,scenario_038,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child refundable CTC and a $3,000 phase-in threshold. For 2026 the earned-income-limited refundable CTC is $2,909.35, while the EITC is $7,316 rather than $7,354."
+us,scenario_038,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied obsolete refundable CTC rules of $1,000 per child and a $3,000 earnings threshold, limiting the credit to $2,000. The applicable refundable CTC is $2,909.35, and the exact two-child EITC plateau amount is $7,316 rather than the estimated $7,300."
+us,scenario_038,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned only $4,100 of EITC despite this joint household being on the two-child plateau, where the credit is $7,316. It also called the full $4,000 CTC refundable instead of applying the refundable earned-income limitation, which yields $2,909.35."
+us,scenario_038,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used only $2,000 of refundable CTC instead of the earned-income-limited $2,909.35. It also rounded the EITC to $7,300 rather than applying the exact 2026 plateau value of $7,316."
+us,scenario_038,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete refundable CTC amount of $1,000 per child, producing $2,000 instead of $2,909.35. It also estimated the EITC at $7,250 rather than applying the $7,316 two-child plateau maximum."
+us,scenario_038,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The unexplained total implies that the model did not correctly combine the two governing components. The required computation is $7,316 of EITC plus $2,909.35 of refundable CTC, totaling $10,225.35."
+us,scenario_038,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model named the correct two credits but understated their combined value by $40.35 without providing a component calculation. The exact components are $7,316 of EITC and $2,909.35 of refundable CTC."
+us,scenario_038,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_038,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model named EITC and refundable CTC but did not calculate either component and understated their sum. The governing calculations yield $7,316 and $2,909.35 respectively, not $8,724 in total."
+us,scenario_038,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that low earnings prevent refundable credits. Two qualifying children and the household's earned income generate a $7,316 EITC and $2,909.35 refundable CTC even though regular income tax is zero."
+us,scenario_038,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model correctly calculated the $7,316 EITC but treated the $1,800-per-child refundable CTC ceiling as an automatic $3,600 payment. The earned-income phase-in binds first, limiting refundable CTC to $2,909.35."
+us,scenario_038,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used $8,231 as the two-child EITC maximum, but the applicable 2026 maximum is $7,316. It also used gross wages plus gross self-employment income in the refundable CTC phase-in, producing $3,073.95 instead of subtracting half of self-employment tax and obtaining $2,909.35."
+us,scenario_038,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model correctly used the $7,316 EITC but understated refundable CTC as $2,731.32. Applying 15% to earned income above $2,500 after the deductible half of self-employment tax yields $2,909.35."
+us,scenario_038,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model wrongly treated refundable credits as requiring positive base income tax or additional unlisted facts. The listed earned income and two qualifying children directly establish a $7,316 EITC and $2,909.35 refundable CTC."
+us,scenario_038,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model imposed an obsolete $2,000 aggregate ACTC limit instead of applying the 2026 refundable CTC phase-in, which yields $2,909.35. It also approximated the EITC at $7,312 rather than using the exact $7,316 maximum."
+us,scenario_038,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied obsolete post-TCJA-reversion assumptions: a $1,000-per-child ACTC and a $3,000 earnings threshold. The applicable refundable CTC is $2,909.35, and the EITC maximum is $7,316 rather than $7,330."
+us,scenario_038,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used obsolete $1,000-per-child refundable CTC rules and a $3,000 phase-in threshold, yielding $2,000 rather than $2,909.35. It also used gross earned income and an estimated $7,338 EITC instead of the exact $7,316 plateau amount."
+us,scenario_038,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model overestimated the EITC plateau amount as roughly $7,422 instead of $7,316. It also understated refundable CTC by using an imprecise earned-income figure; after deducting half of self-employment tax, the 15% formula yields $2,909.35."
+us,scenario_038,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_038,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly used the $7,316 EITC but calculated refundable CTC from gross wages plus gross self-employment income. Earned income for this calculation reflects the deductible half of self-employment tax, reducing refundable CTC from $3,073.95 to $2,909.35."
+us,scenario_038,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model understated the EITC as roughly $6,500 instead of the $7,316 plateau maximum and treated the $3,400 refundable CTC ceiling as the payment. The earned-income phase-in binds below that ceiling at $2,909.35."
+us,scenario_038,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model calculated refundable CTC essentially correctly but used an estimated EITC maximum of about $7,370. The applicable 2026 two-child plateau amount is exactly $7,316, producing the $10,225.35 total."
+us,scenario_038,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model repeatedly mixed three-child and two-child EITC maxima, used a non-joint phaseout threshold, and ultimately submitted a number inconsistent with every total in its reasoning. For this joint filer, no EITC phaseout applies: the EITC is $7,316, and the refundable CTC is $2,909.35 after deducting half of self-employment tax from earned income."
+us,scenario_038,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated $22,993 as almost fully phasing out the two-child EITC to $46. Married filing jointly shifts the phaseout threshold upward, leaving this household on the $7,316 plateau; refundable CTC is $2,909.35 rather than the rounded $3,000."
 us,scenario_038,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted nonqualification without applying either qualifying pathway. PolicyEngine derives categorical eligibility from the household's $7,286.94 annual SNAP benefit and independently derives income eligibility because the school-meal FPG ratio is 0.70, below 1.30."
 us,scenario_038,free_school_meals_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model correctly computed gross income below 130% FPG but then incorrectly imposed SNAP receipt or direct certification as an additional requirement and treated an unlisted SNAP input as proof of no SNAP participation. Income at or below 130% FPG independently qualifies the children for free meals, and PolicyEngine also computes $7,286.94 of SNAP under the assumed-take-up instruction, which supplies categorical eligibility and produces positive meal support."
 us,scenario_038,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -2449,36 +2576,38 @@ us,scenario_038,self_employment_tax,gpt-5.4-nano,llm_error,thresholds_rates,Fals
 us,scenario_038,self_employment_tax,grok-4.3,llm_error,other,False,"The model rounded an exact annual tax calculation down to $2,192 without supporting arithmetic. The 92.35% base is $14,337.34, and applying 15.3% produces $2,193.61."
 us,scenario_038,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self-employment-tax output or explanation, so the required numeric value was missing."
 us,scenario_038,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly fed the deduction for one-half of self-employment tax back into the self-employment-tax base. That deduction reduces adjusted gross income for income-tax purposes only; self-employment tax remains 15.3% of $14,337.34, or $2,193.61."
-us,scenario_038,snap,claude-fable-5,llm_error,other,False,"The model derived an annual benefit near $7,236 but submitted $4,200, so its numeric output does not follow its own calculation. It also used estimated rounded parameters instead of the $994 maximum allotment and $1,309.83 net income."
-us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented mortgage, property-tax, insurance, and utility expenses from the mortgage balance despite the instruction that unlisted expenses are zero. It then abandoned its own $7,772.16 calculation and submitted an unsupported $84 monthly benefit."
-us,scenario_038,snap,claude-opus-4.7,llm_error,other,False,"The model's stated formula produces roughly $6,480 annually, but it switched without arithmetic support to $759 per month and submitted $9,114. It also used estimated deduction parameters instead of the traced $1,309.83 net income."
-us,scenario_038,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model estimated the standard deduction and resulting contribution instead of applying the traced monthly net income of $1,309.83. With the $994 maximum allotment, the correct monthly formula yields $601.30 rather than $654."
-us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,The model replaced the applicable $994 maximum allotment and $392.70 expected contribution with an unsupported estimate of $564 per month. The required monthly benefit is $601.30.
-us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the FY2025 maximum allotment of $975 instead of the applicable $994 amount and rounded the monthly benefit under those stale parameters. It also used a $204 standard deduction rather than the deduction embedded in the traced $1,309.83 net income."
-us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by 20% before computing gross income and then invoked unlisted dependent-care and medical deductions. SNAP applies the earned-income deduction in the net-income calculation, which produces $1,309.83 monthly net income and a $601.30 benefit."
-us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an estimated $1,050 maximum allotment instead of $994. That $56 monthly overstatement drives its benefit above the traced $601.30 monthly amount."
-us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented a $310 monthly dependent deduction even though no dependent-care expense was listed. That unsupported deduction reduced net income below $1,309.83 and overstated the benefit."
-us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $9,000 implies a $750 monthly allotment after contribution, not the $601.30 produced by the $994 maximum and $392.70 contribution. The model supplied no parameter calculation supporting that larger amount."
-us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model stated the general maximum-minus-30%-of-net-income formula but did not apply the actual $994 maximum allotment and $1,309.83 net income. Those inputs yield $7,286.94 annually, not $6,878."
-us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model rounded the maximum allotment down to about $990 and net income up to $1,333. The applicable values are $994 and $1,309.83, producing $601.30 per month rather than $590."
-us,scenario_038,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The $6,276 answer implies only $523 per month despite eligibility and a traced monthly benefit of $601.30. The model omitted the maximum-allotment and expected-contribution computation needed to support its lower figure."
-us,scenario_038,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $9,708 answer implies $809 per month, exceeding the $601.30 obtained after subtracting the $392.70 contribution from the $994 maximum. The model gave no deduction or parameter calculation that supports its figure."
-us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated maximum allotment of $1,020 rather than $994 and an estimated standard deduction. Applying the traced $1,309.83 net income gives a $392.70 contribution and $601.30 monthly benefit."
-us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly denied a positive benefit despite gross income, net income, and assets satisfying the tests and the household qualifying categorically through TANF non-cash assistance. Eligibility leads to a $601.30 monthly allotment."
-us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly treated the household as receiving no SNAP. The household passes the income and asset conditions and has TANF non-cash categorical eligibility, after which the benefit formula yields $7,286.94 annually."
-us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model used an obsolete or unrelated four-person maximum allotment of $879 instead of $994. Its net-income estimate was close, so the $115 monthly maximum-allotment error accounts for the large understatement."
-us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied rounded or estimated deduction and allotment parameters, producing $609.74 per month instead of $601.30. The exact calculation uses $994 minus 30% of $1,309.83."
-us,scenario_038,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model rounded the monthly benefit to $601 before annualizing. The traced monthly amount is $601.30, and preserving that computation across the year produces $7,286.94 rather than $7,212."
-us,scenario_038,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model discarded the cents in the computed monthly allotment and annualized $601. The engine's monthly calculation retains the underlying precision, yielding $7,286.94 annually."
-us,scenario_038,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model misread the no-inference instruction as requiring a zero whenever program parameters were not listed among household facts. SNAP parameters still apply, and the household's TANF non-cash categorical eligibility plus its income and assets produces a positive benefit."
-us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"The model approximated the maximum allotment as $1,000 instead of using $994. Although its net-income estimate was close, that parameter substitution produced $608 per month rather than $601.30."
-us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an annual maximum allotment of about $12,337, equivalent to roughly $1,028 monthly, instead of $994. It also estimated an excessive annual standard deduction, so neither its net income nor maximum matches the traced calculation."
-us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"The model rounded the maximum allotment to about $1,000 and used estimated net income of $1,316. The exact $994 maximum and $1,309.83 net income yield $601.30 monthly, not $605."
-us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $1,020 maximum allotment instead of $994 and then rounded the monthly result down. The applicable maximum less the $392.70 contribution is $601.30 per month."
-us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,The model calculated net income essentially correctly but used a $997 maximum allotment instead of $994. That $3 monthly parameter error accounts for nearly all of the difference.
-us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"The model substituted an estimated $1,050 maximum allotment for the applicable $994 amount. With net income of $1,309.83, the expected contribution is $392.70 and the monthly benefit is $601.30, not about $651."
-us,scenario_038,snap,qwen-3.7-max,llm_error,other,False,"The model's explicit calculation produced about $572.54 monthly, but it then replaced that result with an unsupported $879 monthly amount based on unspecified shelter adjustments. No shelter expense was listed, and the submitted $10,548 does not follow its own formula."
-us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced net income to zero through an unsupported shelter deduction even though no shelter payment or utility expense was listed. Net income is $1,309.83, so the household receives the $994 maximum less a $392.70 contribution rather than the maximum allotment."
+us,scenario_038,snap,claude-fable-5,llm_error,other,False,"The model calculated an annual benefit near $7,236 but submitted $4,200, so it failed to carry its own SNAP calculation into the output. It also replaced the exact $994 allotment and $1,309.83 net income with estimates."
+us,scenario_038,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a $400 monthly shelter deduction from a mortgage balance even though no mortgage payment, taxes, insurance, or utility expense was listed. It then abandoned its own $7,772.16 calculation and submitted an unsupported $1,008."
+us,scenario_038,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model’s stated formula produced about $6,480, but it changed the implied monthly benefit to $759 and submitted $9,114 without a supporting computation. It also used estimated deductions instead of the engine’s $1,309.83 net income."
+us,scenario_038,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an approximate standard deduction and an approximate benefit calculation instead of applying the exact $994 maximum allotment to $1,309.83 of net monthly income. That parameter estimation inflated the annual result to $7,848."
+us,scenario_038,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model asserted a $564 monthly benefit without deriving it from the applicable maximum allotment and net income. The engine calculation uses the $994 maximum and its expected-contribution calculation, producing $7,286.94 annually."
+us,scenario_038,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the FY2025 $975 maximum allotment rather than the applicable $994 amount and used a $204 standard deduction rather than the deduction embedded in the engine’s $1,309.83 net income. Those stale parameters reduced the result to $6,912."
+us,scenario_038,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by 20% before separately discussing the earned-income deduction and also invoked unlisted dependent-care and medical deductions. SNAP net income is $1,309.83 under the supplied facts, not the model’s $900–$1,000 range."
+us,scenario_038,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model substituted a $1,050 maximum allotment for the applicable $994 maximum. Its $208 standard deduction also left net income at $1,325 instead of $1,309.83."
+us,scenario_038,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,The model invented a $310 monthly dependent deduction despite no dependent-care expense being listed. It also used a 2024 maximum allotment rather than the applicable 2026 $994 amount.
+us,scenario_038,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $9,000 implies a $750 monthly allotment, which does not follow from the $994 maximum less the expected contribution based on $1,309.83 net income. The model supplied no parameter-level derivation supporting that amount."
+us,scenario_038,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model named the correct max-allotment-minus-30%-of-net-income structure but used values that yielded $6,878 instead of applying the engine’s $994 maximum and $1,309.83 net income. Its unexplained parameter estimates caused the shortfall."
+us,scenario_038,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model rounded the maximum allotment to about $990 and net income to $1,333. The applicable values are $994 and $1,309.83, so its $590 monthly estimate understated the benefit."
+us,scenario_038,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $6,276 implies only $523 per month despite the household qualifying with $1,309.83 net income and a $994 maximum allotment. The model omitted the calculation and used an unsupported allotment or contribution."
+us,scenario_038,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted $9,708 implies $809 per month, far above the amount produced by subtracting the household’s expected contribution from the $994 maximum. The model gave no deduction or parameter calculation supporting that result."
+us,scenario_038,snap,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $1,020 maximum allotment instead of $994 and a $220 standard deduction that produced the wrong net income. Those parameter substitutions inflated the annual benefit to $7,513.68."
+us,scenario_038,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly denied a positive benefit even though the household passes the gross-income, net-income, and asset requirements and is categorically eligible through TANF non-cash assistance. Those rules produce a substantial allotment rather than zero."
+us,scenario_038,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the household as receiving no SNAP despite categorical eligibility through TANF non-cash assistance and income and assets within the applicable limits. It therefore skipped the maximum-allotment-minus-contribution calculation entirely.
+us,scenario_038,snap,gpt-5.5,llm_error,thresholds_rates,False,The model used an $879 maximum allotment instead of the applicable $994 maximum. That single stale parameter substantially understated the monthly and annual benefit.
+us,scenario_038,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model described the correct general formula but did not apply the exact engine parameters, producing $7,316.88 rather than $7,286.94. Its answer reflects a small maximum-allotment, deduction, or rounding substitution."
+us,scenario_038,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model rounded the monthly benefit to $601 before multiplying by 12. PolicyEngine annualizes its computed benefit without replacing the engine result with that whole-dollar monthly shortcut, yielding $7,286.94."
+us,scenario_038,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model converted a rounded $601 monthly estimate directly to $7,212. It failed to preserve the engine’s precise benefit calculation in the annual output."
+us,scenario_038,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model misapplied the prompt’s no-inference rule by treating missing program parameters as a zero benefit. SNAP rules supply those parameters, and the household is categorically eligible and passes the income and asset tests."
+us,scenario_038,snap,grok-4.5,llm_error,thresholds_rates,False,"The model rounded the maximum allotment to about $1,000 and the monthly result to about $608 instead of applying the exact $994 maximum and engine contribution calculation. Those approximations produced $7,280 rather than $7,286.94."
+us,scenario_038,snap,grok-4.6,llm_error,thresholds_rates,False,"The model used a projected $993 maximum, rounded net income to $1,310, and rounded the resulting allotment to $600 per month. The required calculation preserves the applicable parameters and engine precision."
+us,scenario_038,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an annual maximum allotment of about $12,337, equivalent to roughly $1,028 monthly, rather than the applicable $994 monthly maximum. Its annualized deduction estimates also failed to reproduce the engine’s $1,309.83 monthly net income."
+us,scenario_038,snap,inkling,llm_error,thresholds_rates,False,"The model rounded the maximum allotment to about $1,000 and net income to about $1,316, then rounded the benefit to $605 monthly. Those estimates do not reproduce the exact engine calculation."
+us,scenario_038,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $1,020 maximum allotment instead of $994 and a $208 standard deduction that left net income at $1,325. It then rounded the monthly allotment down, compounding the parameter error."
+us,scenario_038,snap,kimi-k3,llm_error,thresholds_rates,False,"The model used a $997 maximum allotment rather than $994. Although its net-income estimate nearly matched the engine’s $1,309.83, the wrong maximum and its annualization produced $7,248.48."
+us,scenario_038,snap,minimax-m3,llm_error,thresholds_rates,False,"The model substituted a $1,050 maximum allotment for the applicable $994 amount and estimated net income at $1,329 instead of $1,309.83. The overstated maximum drove the $7,800 result."
+us,scenario_038,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used an estimated $207 standard deduction, producing $1,326 of net income rather than $1,309.83. It also rounded the contribution and benefit instead of preserving the engine calculation."
+us,scenario_038,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly computed about $572.54 from its initial assumptions, then introduced unspecified shelter and other adjustments and replaced that result with $879 per month. No shelter expense was listed, so that increase has no allowable deduction behind it."
+us,scenario_038,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced net income to zero through a shelter deduction even though no shelter payment or utility expense was listed. Net income is $1,309.83, so the household does not receive the maximum allotment."
 us,scenario_038,spouse_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model applied only the CHIP age and income conditions and skipped the prerequisite that the person not qualify for Medicaid. At age 18, the spouse is Medicaid-eligible under the OLDER_CHILD category, which makes the spouse ineligible for CHIP."
 us,scenario_038,spouse_chip_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model treated the absence of listed Medicaid coverage as the absence of Medicaid eligibility, despite the question asking about eligibility rather than enrollment. The spouse qualifies for Medicaid under the OLDER_CHILD category, and that eligibility bars CHIP even though the spouse is under 19 and below the CHIP income limit."
 us,scenario_038,spouse_chip_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model concluded that satisfying the CHIP age and income ceiling was sufficient and never evaluated Medicaid eligibility first. The spouse is Medicaid-eligible under the OLDER_CHILD category, so the mutually exclusive CHIP pathway yields No."
@@ -2499,62 +2628,66 @@ us,scenario_038,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error
 us,scenario_038,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model substituted a $4,500-per-person exemption scheme and a 1.75% lowest bracket, producing $3,896 of taxable income. Those parameters do not govern this 2026 Louisiana calculation; the applicable joint-filer treatment leaves the household below the taxable-income threshold."
 us,scenario_038,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required output contract rather than completing the Louisiana tax calculation.
 us,scenario_038,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used an obsolete graduated 2% and 4% rate schedule together with an exemption-only taxable-income calculation. The applicable 2026 Louisiana joint-filer calculation leaves zero taxable income, so neither the $10,896 base nor the graduated-bracket tax applies."
-us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model treated the household as receiving an approximately $7,478 maximum federal EITC instead of calculating the applicable federal EITC of $7,316 from the controlling earned-income and AGI amounts. Applying Louisiana’s 5% rate to the correct federal credit yields $365.80, not $374."
-us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no refundable state EITC. Louisiana provides a refundable credit equal to 5% of the $7,316 federal EITC, producing $365.80."
-us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model reduced the federal EITC to approximately $3,680 despite this joint return with two qualifying children receiving $7,316. It consequently halved the base for Louisiana’s 5% refundable EITC and reported $184 instead of $365.80."
-us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model contradicted its own near-plateau analysis by assigning a federal EITC of only about $2,640. The applicable federal EITC is $7,316, whose 5% Louisiana credit is $365.80."
-us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model rounded the federal EITC base to about $7,000 instead of calculating it as $7,316. Louisiana’s 5% refundable EITC is therefore $365.80, not $350."
-us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly claimed Louisiana eliminated its refundable EITC. The applicable Louisiana refundable EITC is 5% of the $7,316 federal credit, or $365.80."
-us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly classified Louisiana’s EITC as nonrefundable. It is included as a refundable state credit at 5% of the household’s $7,316 federal EITC, yielding $365.80."
-us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used a federal EITC of $7,354 instead of $7,316. Although it applied Louisiana’s 5% rate correctly, the wrong federal credit base produced $367.70 rather than $365.80."
-us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model omitted Louisiana’s refundable EITC. This household receives 5% of its $7,316 federal EITC as a Louisiana refundable credit, totaling $365.80."
-us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model used $7,250 rather than $7,316 as the federal EITC base. At Louisiana’s correctly identified 5% rate, the state refundable credit is $365.80 rather than $362.50."
-us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted $353.43 implies an incorrect federal EITC base of $7,068.60 at Louisiana’s 5% rate. The federal EITC is $7,316, producing a Louisiana refundable credit of $365.80."
-us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The $350 answer applies Louisiana’s 5% credit to an assumed $7,000 federal EITC rather than the computed $7,316 amount. The correct state refundable credit is $365.80."
-us,scenario_038,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly classified Louisiana’s EITC as nonrefundable and made its availability depend on state tax liability. Louisiana’s refundable EITC equals 5% of the $7,316 federal EITC, so it contributes $365.80 even beyond state liability."
-us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model omitted the applicable Louisiana refundable EITC. The household’s $7,316 federal EITC generates a $365.80 Louisiana refundable credit at the 5% state rate."
-us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model incorrectly treated the household’s income level as producing no Louisiana refundable credit. Its $7,316 federal EITC qualifies for Louisiana’s 5% refundable match, yielding $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model treated the lack of separately listed credit inputs as eliminating Louisiana’s EITC. The credit follows automatically from the computed $7,316 federal EITC, and Louisiana refunds 5% of that amount, or $365.80."
-us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model used an inflated federal EITC of $8,231 rather than $7,316. Applying Louisiana’s 5% rate to the correct federal amount gives $365.80, not $411.55."
-us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model used a 3.5% Louisiana EITC rate instead of 5%. Applying the correct 5% rate to the correctly identified $7,316 federal EITC yields $365.80."
-us,scenario_038,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model failed to trigger Louisiana’s refundable EITC. The household’s federal EITC is $7,316, and Louisiana’s 5% refundable credit therefore equals $365.80."
-us,scenario_038,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Louisiana has no applicable state EITC. Louisiana refunds 5% of this household’s $7,316 federal EITC, producing $365.80."
-us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model incorrectly concluded that there was no Louisiana EITC match because no qualifying expenses were listed. The Louisiana refundable EITC depends on the federal EITC, not such expenses, and equals 5% of $7,316, or $365.80."
-us,scenario_038,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model estimated the federal EITC as $7,422 rather than calculating $7,316. Louisiana’s 5% refundable share of the correct federal credit is $365.80, not $371."
-us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract."
-us,scenario_038,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model used an unsupported $6,500 federal EITC base instead of $7,316. Applying Louisiana’s 5% rate to the correct base yields $365.80 rather than $325."
-us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model approximated the household’s federal EITC as the $7,430 maximum instead of calculating the applicable $7,316 credit from earned income and AGI. Louisiana’s 5% refundable share is consequently $365.80, not $371.50."
-us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model omitted Louisiana’s applicable refundable EITC. Louisiana refunds 5% of the household’s $7,316 federal EITC, producing $365.80."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included $25,950 of estate income in AGI and consequently used excessive provisional income for Social Security taxation. It also omitted the $3,319.81 qualified business income deduction and incorrectly switched to qualifying-surviving-spouse thresholds and deductions despite no qualifying child."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included estate income and assumed 85% of Social Security was taxable instead of deriving $10,129.10 of taxable benefits. It then invented disability-based deduction and credit treatment that reduced a positive $1,345.51 regular tax liability to zero, while omitting the QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly selected single filing status but incorrectly included $25,950 of estate income, which drove taxable Social Security to the 85% cap rather than $10,129.10. It also omitted the $3,319.81 QBI deduction, overstating taxable income by more than $46,000."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated the filer as a qualifying surviving spouse and applied joint Social Security thresholds, a joint standard deduction, and joint brackets despite the absence of a qualifying child. It also included estate income in provisional income and omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model included estate income in federal AGI, causing it to tax 85% of Social Security instead of $10,129.10. It omitted the $3,319.81 QBI deduction and therefore taxed roughly $62,500 rather than $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse filing treatment without a qualifying child, using joint Social Security thresholds, deduction, and brackets. It also included estate income and omitted the $3,319.81 QBI deduction, whereas the single-filer computation has AGI of $32,699.07 and taxable income of $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model included $25,950 of estate income and simply treated 85% of Social Security as taxable instead of applying the worksheet that yields $10,129.10. It additionally used qualifying-surviving-spouse parameters and omitted the QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model included estate income, taxed 85% of Social Security, and omitted the QBI deduction. It compounded those errors by using an unsupported $8,800 deduction and pre-TCJA-style 10% and 15% rates instead of the applicable 2026 schedule and $16,100 standard deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model included estate income and taxed 85% of Social Security, inflating AGI from $32,699.07 to $79,209.25. It then used obsolete 15% and 25% brackets and an estimated deduction/exemption combination while omitting the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies that the model combined estate income with an excessive taxable portion of Social Security and failed to subtract the QBI deduction. The correct computation excludes estate income from AGI, includes $10,129.10 of taxable Social Security, and leaves $13,279.25 taxable."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly used AGI of about $79,209, which includes estate income and the maximum 85% taxable Social Security amount. The correct AGI is $32,699.07, followed by the $16,100 standard deduction and $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model included estate income and taxed 85% of Social Security, overstating AGI by $46,510.18. It also used a post-TCJA-expiration deduction and personal-exemption assumption and omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $79,209.25 AGI incorrectly includes estate income and $30,689.25 of taxable Social Security. The applicable derivation uses $10,129.10 of taxable Social Security, AGI of $32,699.07, a $16,100 standard deduction, and a $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model included estate income and taxed the maximum 85% of Social Security, producing an AGI around $79,209 rather than $32,699.07. It also failed to apply the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model put estate income into provisional income, pushing taxable Social Security to $30,689.25 rather than $10,129.10. It also used an estimated standard deduction and omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and surviving-spouse treatment eliminated taxable income. After excluding estate income, including $10,129.10 of taxable Social Security, and applying the standard and QBI deductions, taxable income remains $13,279.25 and generates $1,345.51 of tax."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and unidentified nonrefundable credits eliminated the liability, but no such credits apply. The correct deductions leave $13,279.25 of taxable income, so regular tax remains $1,345.51."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included estate income, treated 85% of Social Security as taxable, and used qualifying-surviving-spouse brackets and a $32,200 deduction without a qualifying child. It also omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used qualifying-surviving-spouse joint treatment and included estate income in the Social Security provisional-income calculation. That produced $25,186.63 of taxable Social Security instead of $10,129.10, and it omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied joint Social Security thresholds, a joint standard deduction, and joint brackets under qualifying-surviving-spouse treatment despite no qualifying child. It also included estate income and omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model included estate income and excessive taxable Social Security in its $79,209 AGI. It then invented an age-61 additional deduction and senior deduction, used joint filing treatment, and omitted the actual $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $8,000 estimate is consistent with including estate income and an excessive taxable portion of Social Security while failing to apply the QBI deduction. The correct income sequence yields AGI of $32,699.07 and taxable income of $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model included estate income, which pushed Social Security to the 85% taxable cap rather than $10,129.10. It also applied projected post-TCJA-expiration deductions and 15% and 25% brackets and omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and consequently taxed $30,689 of Social Security instead of $10,129.10. It also omitted the $3,319.81 QBI deduction, producing taxable income of $63,789 instead of $13,279.25."
-us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model included estate income and taxed 85% of Social Security, yielding AGI of about $79,209 rather than $32,699.07. Although it used the correct $16,100 standard deduction, it omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric value or explanation for the requested output, so its response could not be parsed or evaluated substantively."
-us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly applied single status and the $16,100 standard deduction but incorrectly included estate income, causing taxable Social Security to reach $30,689.25 instead of $10,129.10. It also omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model counted the full Social Security benefit in gross income and then added an 85% taxable amount again, double counting Social Security. It also included estate income, invented additional deductions for being age 61 and disabled, and omitted the $3,319.81 QBI deduction."
-us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"Despite repeated recalculations, the model retained estate income in provisional income and used qualifying-surviving-spouse joint parameters, producing about $25,197 of taxable Social Security rather than $10,129.10. It also omitted the $3,319.81 QBI deduction and introduced unlisted mortgage interest and state-tax estimates into its abandoned itemization analysis."
-us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included estate income, used half the self-employment loss instead of the full $6,260 above-the-line deduction, and omitted the $3,319.81 QBI deduction. It also invented a credit for other dependents from pension income and an AMT floor, neither of which applies."
+us,scenario_038,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model treated the household as receiving the projected maximum federal EITC of about $7,478 instead of computing the applicable $7,316 federal EITC. Applying Louisiana's 5% rate to the inflated federal amount produced $374 rather than $365.80."
+us,scenario_038,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Louisiana has no refundable EITC. This household receives a refundable Louisiana credit equal to 5% of its $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model reduced the federal EITC to approximately $3,680 at an income where the applicable federal credit is $7,316. Its federal EITC phase-in, plateau, or phaseout calculation was therefore wrong, causing the Louisiana 5% credit to be understated."
+us,scenario_038,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model contradicted its own observation that the household was near the federal EITC plateau by substituting an unsupported federal EITC of $2,640. The applicable federal EITC is $7,316, whose 5% Louisiana match is $365.80."
+us,scenario_038,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model used a rounded federal EITC of about $7,000 instead of calculating the traced $7,316 amount. Louisiana's 5% match therefore equals $365.80, not the rounded $350."
+us,scenario_038,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Louisiana eliminated its refundable EITC. The applicable Louisiana EITC is refundable and equals 5% of the household's $7,316 federal EITC."
+us,scenario_038,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly classified Louisiana's EITC as nonrefundable. Its refundable 5% match of the $7,316 federal EITC yields $365.80."
+us,scenario_038,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used a federal EITC of $7,354 instead of $7,316. The 5% Louisiana calculation was arithmetically correct for its input, but its federal EITC computation overstated the credit by $38."
+us,scenario_038,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC. The household's $7,316 federal EITC generates a $365.80 Louisiana refundable credit at the 5% state rate."
+us,scenario_038,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model substituted a $7,250 federal EITC for the applicable $7,316 amount. Although it applied the correct 5% Louisiana rate, the wrong federal-credit base produced $362.50."
+us,scenario_038,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted $353.43 implies a federal EITC base of $7,068.60 at Louisiana's 5% rate. The correct federal base is $7,316, producing $365.80."
+us,scenario_038,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model recognized the Louisiana EITC but used a rounded $7,000 federal-credit base. The computed federal EITC is $7,316, so the state credit is $365.80."
+us,scenario_038,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated the Louisiana EITC as nonrefundable and tied it to available state tax liability. The credit is refundable, and 5% of the $7,316 federal EITC is $365.80 regardless of zero pre-credit state liability."
+us,scenario_038,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model omitted the refundable Louisiana EITC applicable to this household. Its 5% match of the $7,316 federal EITC yields $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no positive refundable Louisiana credit applies at this income. The household qualifies for a $365.80 Louisiana EITC based on 5% of its $7,316 federal EITC."
+us,scenario_038,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the absence of separately listed credit inputs as eliminating the Louisiana EITC. Eligibility and the amount follow from the household's earned income and qualifying children, producing 5% of the $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model used an $8,231 federal EITC, overstating the applicable federal credit of $7,316. Applying the correct Louisiana 5% rate to the correct federal base yields $365.80."
+us,scenario_038,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied a 3.5% Louisiana EITC percentage instead of the applicable 5% rate. Five percent of the correctly identified $7,316 federal EITC is $365.80."
+us,scenario_038,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to trigger Louisiana's refundable EITC from the household's earnings and two qualifying children. The credit equals 5% of the $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Louisiana has no state EITC applicable to the household. Louisiana provides a refundable 5% match of the $7,316 federal EITC, yielding $365.80."
+us,scenario_038,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,"The model incorrectly denied the existence of an applicable refundable Louisiana EITC. The household receives $365.80, calculated as 5% of its $7,316 federal EITC."
+us,scenario_038,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly required a separate qualifying expense or explicit EITC-match input. The household's earnings and qualifying children generate a $7,316 federal EITC and therefore a refundable $365.80 Louisiana match."
+us,scenario_038,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model estimated the federal EITC as $7,422 instead of computing $7,316. Its correct 5% state rate applied to the overstated federal base produced approximately $371 rather than $365.80."
+us,scenario_038,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for state_refundable_credits, violating the required output contract."
+us,scenario_038,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model used an unsupported $6,500 federal EITC base instead of the computed $7,316. Applying Louisiana's 5% rate to the correct base yields $365.80."
+us,scenario_038,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model incorrectly conditioned refundable-credit eligibility on positive Louisiana taxable income or school-readiness expenses. The refundable Louisiana EITC applies independently and equals 5% of the $7,316 federal EITC, or $365.80."
+us,scenario_038,state_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model treated the household as receiving an estimated maximum federal EITC of $7,430 instead of computing the applicable $7,316 credit. The correct 5% Louisiana match is therefore $365.80, not $371.50."
+us,scenario_038,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC. The household's federal EITC is $7,316, and Louisiana refunds 5% of that amount, producing $365.80."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It included the $25,950 estate-income input in federal AGI and then applied qualifying-surviving-spouse Social Security thresholds and deductions. The correct computation excludes that estate income from AGI, includes $10,129.10 of taxable Social Security, and deducts $3,319.81 of QBI."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It included the estate income and 85% of Social Security in AGI, then invented an elderly/disabled exclusion or credit that eliminated all tax. Disability at age 61 does not create that standard-deduction addition, and the actual taxable income after the standard and QBI deductions is $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It treated the $25,950 estate-income input as federal gross income, causing provisional income to make 85% of Social Security taxable. It also omitted the $3,319.81 QBI deduction; the correct taxable Social Security amount is $10,129.10."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It included estate income in AGI and used qualifying-surviving-spouse thresholds and the joint standard deduction. The engine instead derives $32,699.07 of AGI, applies the $16,100 standard deduction and $3,319.81 QBI deduction, and reaches $13,279.25 of taxable income."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It counted the estate income in AGI and assumed 85% of Social Security was taxable, while omitting the QBI deduction. Those errors inflated AGI from $32,699.07 to about $79,209."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It treated the estate income as taxable AGI and applied qualifying-surviving-spouse Social Security thresholds and a joint standard deduction. It also failed to subtract the $3,319.81 QBI deduction, leaving taxable income far above the correct $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It included estate income in federal AGI, treated 85% of Social Security as taxable, and used a projected joint standard deduction. The correct derivation includes only $10,129.10 of taxable Social Security and also deducts $3,319.81 for QBI."
+us,scenario_039,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It combined an inflated $79,209 AGI with an obsolete post-TCJA-sunset standard deduction and 15% bracket schedule. The correct 2026 computation uses $32,699.07 of AGI, $19,419.81 of total deductions, and the standard rate schedule on $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It counted estate income and 85% of Social Security in AGI, then used an inapplicable TCJA-expiration schedule with a personal exemption. It omitted the $3,319.81 QBI deduction and taxed $65,459.25 instead of $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its unexplained $5,354 estimate is inconsistent with the traced taxable income of $13,279.25 and tax of $1,345.51. The shortcut omitted the estate-income treatment, $10,129.10 taxable-Social-Security calculation, or $3,319.81 QBI deduction required by the derivation."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It explicitly used an estimated $79,209 AGI, which counts the estate income and 85% of Social Security. The correct AGI is $32,699.07, followed by $19,419.81 of deductions."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It included estate income and 85% of Social Security in AGI and applied an inapplicable personal exemption and post-TCJA rate schedule. It also omitted the $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It inflated AGI to $79,209.25 by counting estate income and $30,689.25 of Social Security as taxable. It then omitted the $3,319.81 QBI deduction; correct AGI is $32,699.07."
+us,scenario_039,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"It based the liability on approximately $79,209 of AGI, reflecting inclusion of estate income and 85% of Social Security. The correct computation uses $32,699.07 of AGI and both the standard and QBI deductions."
+us,scenario_039,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It included the estate income when computing provisional income, forcing taxable Social Security to its 85% cap and AGI to $79,209.25. Excluding that amount from federal AGI yields $10,129.10 of taxable Social Security, and the model also missed the $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It incorrectly concluded that surviving-spouse or elderly treatment made taxable income nonpositive. The traced deductions leave $13,279.25 of positive taxable income, producing $1,345.51 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It asserted that deductions and unspecified nonrefundable credits eliminated the liability without computing them. No credits reduce the result, and $13,279.25 of taxable income produces $1,345.51 of tax."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It counted estate income in non-Social-Security income, taxed 85% of Social Security, and used a joint surviving-spouse standard deduction. It omitted the $3,319.81 QBI deduction and overstated AGI by more than $46,000."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It included estate income in AGI and applied MFJ Social Security thresholds and deductions. The correct path produces $10,129.10 of taxable Social Security and subtracts a $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It treated the filer under qualifying-surviving-spouse joint rules and included estate income in provisional income. The correct derivation uses $32,699.07 of AGI and $19,419.81 of deductions rather than a $32,200 joint standard deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It inflated AGI to about $79,209 by including estate income and excessive taxable Social Security, then invented an age-61 addition and senior deduction. Age 61 supplies neither deduction; the applicable deductions are the $16,100 standard deduction and $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its $8,000 estimate implies a much larger tax base than the traced $13,279.25 and cites estate income as taxable. It failed to apply the correct estate-income treatment, taxable-Social-Security calculation, and $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It included estate income and 85% of Social Security in AGI, then applied an inapplicable TCJA-sunset standard deduction, personal exemption, and 10%/15%/25% brackets. The correct 2026 schedule applies to only $13,279.25 after the standard and QBI deductions."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It counted estate income and 85% of Social Security in AGI and used an obsolete TCJA-sunset deduction, exemption, and bracket structure. It omitted the $3,319.81 QBI deduction and taxed $65,609 rather than $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It included estate income in provisional income, taxed 85% of Social Security, and omitted the QBI deduction. Those steps produced $63,789 of taxable income instead of $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It counted the estate income and $30,689 of Social Security in AGI, producing $79,209 instead of $32,699.07. It also subtracted only the standard deduction and omitted the $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It supplied no parseable value or explanation for the requested output.
+us,scenario_039,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It included the $25,950 estate-income input in provisional income and AGI, causing $30,689.25 rather than $10,129.10 of Social Security to be taxable. It also omitted the $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It double-counted Social Security while constructing taxable income, incorrectly used the full benefit in provisional income, and invented extra deductions for being age 61 and disabled. The correct calculation includes $10,129.10 of taxable Social Security once and deducts the standard deduction plus QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It included estate income in provisional income, thereby taxing 85% of Social Security and raising AGI to $79,209. It also omitted the $3,319.81 QBI deduction."
+us,scenario_039,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"After several contradictory calculations, it settled on including estate income in provisional income, applying MFJ Social Security thresholds and a joint standard deduction. It also omitted the $3,319.81 QBI deduction; the correct taxable income is $13,279.25."
+us,scenario_039,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It deducted only half the self-employment loss, included estate income and excessive taxable Social Security, and invented a credit for other dependents based on pension income. Pension income does not create a dependent credit; the full $6,260.03 loss, $16,100 standard deduction, and $3,319.81 QBI deduction lead to $1,345.51."
 us,scenario_039,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_039,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly concluded that the head's income was below the expansion threshold and treated disability as compatible with an eligible adult or ABD pathway despite acknowledging the resource-limit problem. The trace places MAGI at 3.68 times FPL, so the expansion pathway fails, and the head qualifies for no ABD or other Medicaid category."
 us,scenario_039,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability as sufficient for Medicaid and failed to apply Virginia's categorical and income tests. The head has no qualifying Medicaid category, and the MAGI expansion pathway fails at 3.68 times FPL."
@@ -2574,36 +2707,38 @@ us,scenario_039,head_medicare_eligible,minimax-m3,llm_error,categorical_eligibil
 us,scenario_039,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented wage earnings and applied the 7.65% employee Social Security and Medicare rate to an unstated payroll-tax base. The household has no listed wage or salary income, while its listed estate, retirement, IRA, pension, and negative self-employment income are not subject to employee payroll tax, yielding $0."
 us,scenario_039,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required structured-output contract."
 us,scenario_039,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model replaced the computed federal AGI of $32,699.07 with approximately $48,520 plus taxable Social Security, leaving $39,770 taxable after its deductions. Applying the traced Virginia subtraction, $8,750 standard deduction, and $930 exemption yields taxable income of $12,889.97, not its inflated base."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a retirement-income exclusion, disability-income exclusion, and retirement credit that eliminate the tax. The applicable computation instead leaves $12,889.97 of Virginia taxable income after the traced subtractions, standard deduction, and personal exemption, producing $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated surviving-spouse status as entitling this taxpayer to a roughly $17,000 married-filing-jointly Virginia standard deduction and began from an overstated $48,520 Virginia AGI. The computation uses Virginia AGI of $22,569.97, an $8,750 deduction, and a $930 exemption."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used $48,520 as Virginia AGI and then introduced an unsupported disability exemption and an unexplained downward adjustment. Virginia AGI is $22,569.97 after $10,129.10 of subtractions, and only the traced $8,750 deduction and $930 exemption reduce it to $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model inferred roughly $40,000 of taxable income from a gross-income reconstruction instead of using the computed federal AGI and Virginia subtractions. The correct Virginia taxable base is $12,889.97, so applying the brackets produces $514.50 rather than $1,990."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated all $36,105 of Social Security as part of federal AGI, invented a $20,000 disabled-retirement subtraction, used a joint standard deduction, and claimed extra exemptions. The trace instead starts at $32,699.07, subtracts $10,129.10, $8,750, and $930, leaving $12,889.97 taxable."
-us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model reconstructed Virginia AGI as $48,520 rather than using $22,569.97 after the traced Virginia subtractions. Its own bracket calculation supports about $2,044, but it then changed the result to $3,400 without any computation, compounding the taxable-base error."
-us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied Virginia deductions directly to $79,209 of reconstructed federal AGI and omitted the $10,129.10 Virginia subtraction. The resulting $69,529 base should instead be $12,889.97 after starting from the traced $32,699.07 AGI."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model assumed that removing taxable Social Security from its reconstructed federal AGI leaves $48,520 of Virginia AGI. The trace leaves $22,569.97 after Virginia subtractions, and the $8,750 deduction plus $930 exemption reduce that to $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer provides no calculation supporting $2,420 and implies a substantially inflated Virginia taxable base. The traced AGI, subtractions, deduction, and exemption produce taxable income of $12,889.97 and tax of $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Although the model names the Social Security subtraction and standard deduction, its $1,990 answer reflects the common $48,520 Virginia-AGI shortcut. The traced Virginia AGI is $22,569.97, leaving only $12,889.97 taxable."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly fixed Virginia AGI at $48,520 by summing non-Social-Security inputs. Virginia AGI is $22,569.97 after the traced subtraction, and it also used incorrect $9,000 and $800 deduction figures instead of $8,750 and $930."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model reconstructed federal AGI as $79,209.25 and consequently calculated $39,590 of Virginia taxable income. The trace starts at $32,699.07 and, after the applicable subtraction, $8,750 deduction, and $930 exemption, leaves $12,889.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $1,990 result implies that the model used about $48,520 of Virginia AGI after excluding Social Security. The traced Virginia AGI is $22,569.97, producing $12,889.97 of taxable income and $514.50 of tax."
-us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated the arithmetic sum of estate income, IRA distributions, pension, and the self-employment loss as $48,520 of Virginia AGI. That bypasses the engine-derived federal AGI of $32,699.07 and $10,129.10 Virginia subtraction, which leave Virginia AGI of $22,569.97."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-based treatment eliminates the taxable base, but age 61 does not erase the remaining Virginia income. The traced deductions and exemption leave $12,889.97 taxable, producing $514.50."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model classified the case as a zero-tax low-income scenario without computing Virginia taxable income. The applicable subtractions and deductions leave $12,889.97 taxable, which generates $514.50 under the Virginia brackets."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $48,520 of purported non-Social-Security federal income as Virginia AGI. The traced Virginia AGI is $22,569.97, and subtracting $8,750 and $930 leaves $12,889.97 rather than $39,090."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model invoked a surviving-spouse or married-filing-jointly standard deduction while still returning a tax far above the traced result. The calculation uses the $8,750 Virginia standard deduction against Virginia AGI of $22,569.97, followed by the $930 exemption."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly applied an estimated $17,500 joint standard deduction and started from an inflated income base. The trace uses Virginia AGI of $22,569.97 and the $8,750 standard deduction, producing $12,889.97 of taxable income."
-us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model relied on an unspecified surviving-spouse deduction while failing to reproduce the traced Virginia AGI. The applicable $8,750 deduction and $930 exemption reduce $22,569.97 to $12,889.97, not the much larger base implied by $1,789 of tax."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that the standard deduction and exemptions offset all taxable income. They reduce Virginia AGI of $22,569.97 to $12,889.97, which remains subject to $514.50 of tax."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used $48,520 as Virginia AGI rather than the traced $22,569.97 and also used an $8,000 standard deduction instead of $8,750. Those errors inflated taxable income from $12,889.97 to $39,590."
-us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model replaced the traced Virginia AGI with the $48,520 arithmetic sum of selected income inputs and used an incorrect $8,000 deduction. The correct subtraction and deduction sequence leaves $12,889.97 taxable."
-us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model reconstructed $48,520 of Virginia AGI from gross income components and used an approximate $8,000 standard deduction. The trace gives $22,569.97 of Virginia AGI and an $8,750 deduction, leaving $12,889.97 taxable."
-us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract."
-us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,750 deduction and $930 exemption but applied them to an erroneous $48,520 Virginia AGI. Applying them to the traced $22,569.97 Virginia AGI yields $12,889.97 of taxable income and $514.50 of tax."
-us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $84,625 gross household total as federal AGI and invented age-65 and surviving-spouse adjustments even though the taxpayer is 61. It then abandoned its own roughly $4,100 calculation and submitted $1,500 without a tax computation."
-us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full Social Security amount as part of federal AGI, mislabeled the $930 personal exemption as the standard deduction, and omitted the actual $8,750 standard deduction. Its submitted $3,260.45 also contradicts its own calculated $1,878.93."
-us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented an $11,898 age-61 exemption and treated the $2,030 pension amount as a nonrefundable credit that offsets tax dollar for dollar. The trace instead applies the $8,750 standard deduction and $930 exemption to $22,569.97 of Virginia AGI, leaving $12,889.97 taxable."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model constructed a $48,520 Virginia income base by directly summing estate, IRA, pension, and self-employment inputs instead of using federal AGI of $32,699.07. Applying the Virginia subtractions, $8,750 standard deduction, and $930 exemption to the correct starting amount yields $12,889.97 of taxable income."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented a Virginia retirement-income credit and disability exclusion that eliminate the tax. The applicable computation instead leaves $12,889.97 taxable after the traced Virginia subtraction, standard deduction, and personal exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated surviving-spouse status as entitlement to a roughly $17,000 married Virginia standard deduction and also began from an unsupported $48,520 income base. The trace applies an $8,750 deduction and $930 exemption to Virginia AGI of $22,569.97."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model replaced the traced federal AGI with $79,209 by summing the listed income inputs and an assumed 85% taxable Social Security amount, then introduced unsupported adjustments to reach its answer. Federal AGI is $32,699.07, and the Virginia subtraction reduces it to $22,569.97 before deductions."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model implicitly retained about $48,520 after excluding Social Security instead of using Virginia AGI of $22,569.97. That overstated base caused it to tax roughly $40,000 rather than the traced $12,889.97."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the $84,625 gross-income total as federal AGI and invented a $20,000 disabled-retirement subtraction, two personal exemptions, and a married standard deduction. The trace starts with $32,699.07 of federal AGI and uses $10,129.10 of Virginia subtractions, an $8,750 deduction, and one $930 exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model constructed federal AGI of about $79,209 and Virginia AGI of $48,520 directly from gross income components. The engine-derived amounts are $32,699.07 and $22,569.97, respectively, so its bracket calculation used a greatly overstated taxable base."
+us,scenario_039,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model taxed federal AGI of $79,209 without applying the $10,129.10 Virginia subtraction. It also used the wrong federal AGI; the traced starting amount is $32,699.07."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model derived a $48,520 Virginia AGI by subtracting assumed taxable Social Security from an invented $79,209 federal AGI. The traced Virginia AGI is $22,569.97, leaving only $12,889.97 taxable after deductions and exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $2,420 answer implies a taxable-income base far above the traced $12,889.97. The model failed to carry federal AGI of $32,699.07 through the $10,129.10 Virginia subtraction, $8,750 deduction, and $930 exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Although the model named the Social Security subtraction and standard deduction, its $1,990 answer implies it used about $39,000 of taxable income. The correct sequence produces $22,569.97 of Virginia AGI and $12,889.97 of taxable income."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $79,209 as federal AGI and $48,520 as Virginia AGI instead of the traced $32,699.07 and $22,569.97. Its altered deduction and exemption amounts compounded the overstatement."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model began with an unsupported $79,209.25 federal AGI and consequently calculated $39,590 of taxable income. The trace begins at $32,699.07 and reaches taxable income of $12,889.97 after Virginia adjustments."
+us,scenario_039,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $1,990 result is consistent with taxing roughly $39,000 rather than the traced $12,889.97. The model did not derive the correct $32,699.07 federal AGI and $22,569.97 Virginia AGI."
+us,scenario_039,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model directly summed listed income into $48,520 of non-Social-Security income and therefore overstated federal AGI as $79,209.25. The traced federal AGI is $32,699.07, and Virginia AGI after subtractions is $22,569.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-based treatment eliminated the taxable base, but age 61 does not produce the claimed result. The traced deductions leave $12,889.97 taxable and therefore a positive bracket tax."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the household as a zero-tax low-income case without calculating Virginia taxable income. The applicable subtractions, standard deduction, and exemption leave $12,889.97 taxable."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $48,520 as non-Social-Security Virginia income instead of the traced Virginia AGI of $22,569.97. This produced $39,090 of taxable income rather than $12,889.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $1,760.18 result implies it retained a much larger income base after excluding Social Security than the traced $22,569.97 Virginia AGI. It failed to use federal AGI of $32,699.07 and the full traced $10,129.10 Virginia subtraction."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied an estimated $17,500 joint standard deduction based on surviving-spouse status while also using an overstated pre-deduction income base. The trace applies an $8,750 standard deduction to Virginia AGI of $22,569.97."
+us,scenario_039,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income far above $12,889.97 despite naming the relevant broad adjustments. The model failed to anchor the calculation to federal AGI of $32,699.07 and Virginia AGI of $22,569.97."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction and exemptions offset all taxable income. They reduce $22,569.97 of Virginia AGI to $12,889.97, not to zero."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used $48,520 as Virginia AGI after subtracting Social Security from an overstated federal AGI. The traced Virginia AGI is $22,569.97, and the correct deduction is $8,750 rather than $8,000."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly built federal AGI of $79,209 and Virginia AGI of $48,520, then analyzed an age/disability deduction phaseout that is not part of the traced result. The correct Virginia AGI is $22,569.97 before the $8,750 deduction and $930 exemption."
+us,scenario_039,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated Virginia AGI as $48,520 rather than $22,569.97 and used an $8,000 standard deduction rather than $8,750. Those errors inflated taxable income from $12,889.97 to $39,590."
+us,scenario_039,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model constructed federal AGI of about $79,209 and Virginia AGI of $48,520 instead of using $32,699.07 and $22,569.97. It also understated the standard deduction as about $8,000 rather than $8,750."
+us,scenario_039,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured answer was missing."
+us,scenario_039,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly stated the Virginia rate schedule but fed it $38,840 of taxable income after constructing an unsupported $48,520 Virginia AGI. The trace yields Virginia AGI of $22,569.97 and taxable income of $12,889.97."
+us,scenario_039,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $84,625 gross total as federal AGI and invented age-65, disability, and surviving-spouse adjustments despite the taxpayer being 61. It then abandoned its own roughly $4,100 computation for an unsupported $1,500 estimate instead of applying the traced Virginia calculation."
+us,scenario_039,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly denied the Virginia subtraction for federally taxable Social Security based on a fabricated $50,000 AGI restriction. The trace applies $10,129.10 of Virginia subtractions, primarily taxable Social Security, to federal AGI of $32,699.07."
+us,scenario_039,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the $84,625 gross household-income sum as federal AGI, confused the $930 personal exemption with the standard deduction, and omitted the actual $8,750 deduction. Its submitted $3,260.45 also contradicts its own stated bracket calculation of $1,878.93."
+us,scenario_039,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented an $11,898 age-61 exemption and treated $2,030 of pension income as a nonrefundable credit that fully offsets tax. Pension income is not a dollar-for-dollar tax credit, and the traced deductions leave $12,889.97 taxable."
 us,scenario_039,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_040,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete TCJA-sunset regime with personal exemptions and only $18,850 of standard and age-based deductions. The applicable 2026 standard deduction and additional deductions for two spouses over 65 exceed the $32,300 AGI, so taxable income is zero rather than $2,850."
 us,scenario_040,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model assumed the TCJA expired and combined personal exemptions with an understated $19,700 standard deduction and elderly add-ons. Under the applicable 2026 deduction rules, total standard and age-based deductions exceed $32,300 of AGI, leaving no taxable income to tax at 10%."
@@ -2633,70 +2768,74 @@ us,scenario_040,state_income_tax_before_refundable_credits,minimax-m3,llm_error,
 us,scenario_040,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model initially included $21,420 of Social Security in Arizona taxable income instead of subtracting federally taxable Social Security, then combined mutually inconsistent tax, exclusion, and credit calculations to reach an amount larger than its stated intermediate liability. After the Social Security subtraction and joint standard deduction, Arizona taxable income and tax are both zero."
 us,scenario_040,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model conflated Arizona’s distinct excise-tax and property-tax credits and treated age plus an asserted $25,000 AGI ceiling as sufficient for a $200 refundable credit. It never applied the specific eligibility requirements for either Arizona credit; this household qualifies for neither, so az_refundable_credits is $0."
 us,scenario_040,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted state_refundable_credits entirely instead of submitting the required numeric value and explanation. The required output is $0 because every state and locality refundable-credit component, including Arizona’s, evaluates to zero."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model treated the $3,753 non-Schedule-D capital-gain input as additional gross income and calculated $16,057 of taxable Social Security instead of $13,655.95. Those errors raised AGI and taxable income above the traced amounts despite correctly applying the $24,150 combined deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used a surviving-spouse deduction and brackets even though no qualifying child is present, so the filer is single. It also included the full Social Security benefit in income and invented a $783 dependent-exemption credit, rather than calculating $13,655.95 of taxable Social Security and applying the $18,150 standard deduction plus $6,000 senior deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded the $29,580 Social Security benefit entirely by treating it as another person's income and omitted the $6,000 senior deduction. The correct return includes $13,655.95 of taxable Social Security and uses total deductions of $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned qualifying-surviving-spouse status and MFJ-equivalent deductions and brackets without a qualifying child. It also overstated taxable Social Security at roughly $25,143; the single-filer computation includes $13,655.95 and applies $24,150 of total deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the 85% maximum, $25,143, as taxable Social Security instead of applying the statutory provisional-income formula, which yields $13,655.95. That overstatement, together with counting the non-Schedule-D gain as gross income, inflated AGI above $43,637.65."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $16,907 by building provisional income from items not included in the traced income base, then omitted the $6,000 senior deduction from its final tax calculation. The correct AGI is $43,637.65 and total deductions are $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded all taxable Social Security and then asserted that the standard deduction eliminated ordinary taxable income, contradicting its own estimate of roughly $16,792 of taxable income. The correct calculation includes $13,655.95 of taxable Social Security and leaves $19,487.65 of taxable income after the $24,150 deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model calculated $16,906.60 of taxable Social Security and used only a $17,425 standard deduction. The traced calculation uses $13,655.95 of taxable Social Security and also applies the $6,000 senior deduction, reducing taxable income to $19,487.65."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied an assumed post-TCJA standard deduction and personal exemption instead of the 2026 $18,150 single-and-aged standard deduction plus the $6,000 senior deduction. It also overstated taxable Social Security at $16,907, producing taxable income far above $19,487.65."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model says it used a head-of-household senior deduction structure, but the filer has no qualifying dependent and files single. Its answer also implies omission of the separate $6,000 senior deduction that brings total deductions to $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from an AGI near $50,581 rather than the traced $43,637.65, reflecting an overstated taxable Social Security or income base. It also failed to incorporate the full $24,150 of standard, aged, and senior deductions needed to reach $19,487.65 of taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a pre-TCJA personal-exemption regime and only $15,150 of deductions. For 2026 the calculation uses the $18,150 single-and-aged standard deduction plus the $6,000 senior deduction, and AGI is $43,637.65 rather than about $50,581."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly used the much larger qualifying-surviving-spouse standard deduction despite the absence of a qualifying child. The filer is single, with $13,655.95 of taxable Social Security and $24,150 of total deductions, leaving $19,487.65 taxable."
-us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's stated AGI of about $50,581 overstates the traced $43,637.65 by using the wrong taxable Social Security or income base. Its $3,584 explanation also conflicts with its submitted $3,090, and neither reflects the $24,150 combined deductions and $19,487.65 taxable income."
-us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated non-Social-Security income and calculated $17,757 of taxable Social Security, yielding AGI of $52,563 instead of $43,637.65. It then omitted the separate $6,000 senior deduction and used only $17,850 of deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that age-related deductions and credits eliminate all taxable income. After the $18,150 standard deduction and $6,000 senior deduction, $19,487.65 remains taxable, and no nonrefundable credit reduces the resulting $1,979.16 tax."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the available deductions and nonrefundable credits as sufficient to reduce liability to zero. The allowed deductions total $24,150, no nonrefundable credit applies, and $19,487.65 of taxable income remains."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used an estimated $33,200 surviving-spouse deduction despite the absence of a qualifying child. The filer is single and receives $18,150 of standard-and-aged deduction plus the $6,000 senior deduction, not a surviving-spouse deduction."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at about $58,949 by using an excessive taxable Social Security amount and an inflated income base. It also taxed qualified dividends and capital gains at 15%, whereas the traced $928 of qualified dividends lies entirely in the 0% band."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly applied qualifying-surviving-spouse deductions and brackets without a qualifying child. The correct single-filer deductions total $24,150 and leave $18,559.65 of ordinary taxable income, not about $6,036."
-us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used a surviving-spouse standard deduction even though no qualifying child exists. Applying single status, $43,637.65 of AGI, and $24,150 of total deductions yields $19,487.65 of taxable income rather than an amount confined to the 10% bracket."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income materially above $19,487.65 because it did not apply the traced income base and full $24,150 deductions. The correct ordinary taxable amount is $18,559.65 after removing $928 of qualified dividends, producing $1,979.16."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an assumed pre-TCJA standard deduction, personal exemption, and 15% ordinary bracket instead of the operative 2026 deduction and rate parameters. It also overstated taxable Social Security at $16,907; the traced amount is $13,655.95 and total deductions are $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $24,205 and used a reverted pre-TCJA deduction and 15% bracket. The correct computation includes $13,655.95 of taxable Social Security, $24,150 of deductions, and the 10%/12% single schedule."
-us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model double-counted the identical farm-income and farm-operations loss fields, reducing income by $2,254 instead of the traced $1,127.29 deduction, while also overstating taxable Social Security and omitting the $6,000 senior deduction. The correct AGI is $43,637.65 and total deductions are $24,150."
-us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required structured answer could not be parsed."
-us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model calculated $24,108.75 of taxable Social Security instead of $13,655.95 and included additional amounts outside the traced gross-income base. Although it correctly identified the $24,150 deductions, its inflated AGI raised taxable income to $33,693.75 rather than $19,487.65."
-us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model applied MFJ Social Security thresholds and a qualifying-surviving-spouse standard deduction despite there being no qualifying child. The taxpayer files single, has $13,655.95 of taxable Social Security, and receives $24,150 of total deductions."
-us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned qualifying-surviving-spouse status and corresponding deductions and brackets without a qualifying child. It also included the tax-exempt IRA distribution in provisional income and overstated taxable Social Security, whereas the traced taxable Social Security amount is $13,655.95."
-us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The submitted $21,824 bears no computational relationship to the household's $43,637.65 AGI, $24,150 deductions, or $19,487.65 taxable income. The model supplied no rule-level derivation and effectively failed to apply the ordinary 10%/12% rate schedule to the traced taxable-income base."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $16,057 instead of $13,655.95 and consequently overstated AGI and taxable income. With the stated $24,150 deductions and $928 qualified-dividend exclusion, ordinary taxable income is $18,559.65, producing $1,979.16."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a surviving-spouse standard deduction despite there being no qualifying child, included Social Security without applying its taxable-benefit formula, and invented a dependent-exemption credit. The filer is single, and the applicable deductions are $18,150 plus the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded the $29,580 Social Security benefit from the head's income by treating it as another person's benefit and omitted the $6,000 senior deduction. PolicyEngine includes $13,655.95 of the benefit in AGI and uses total deductions of $24,150."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse brackets and deductions without a qualifying child and overstated taxable Social Security at $25,143. The correct computation uses single status, $13,655.95 of taxable Social Security, and $24,150 of deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated 85% of the entire Social Security benefit as taxable instead of applying the tiered taxable-benefit formula, which yields $13,655.95. That error raised AGI from $43,637.65 to roughly $58,878 and materially overstated tax."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"Although the model eventually improved its Social Security calculation, it still used $16,907 rather than $13,655.95 and omitted the separate $6,000 senior deduction. Those errors left ordinary taxable income far above the correct $18,559.65."
+us,scenario_042,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded taxable Social Security and then asserted that deductions and preferential rates eliminated all tax, despite its own calculation leaving substantial ordinary taxable income. The correct computation includes $13,655.95 of taxable Social Security and yields $1,979.16 after the $24,150 deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model omitted the $6,000 senior deduction and overstated taxable Social Security at $16,906.60. PolicyEngine uses $43,637.65 of AGI and $24,150 of deductions, not $50,712.60 of AGI and only $17,425 of deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied assumed post-TCJA-expiration personal-exemption rules and omitted the $6,000 senior deduction, while also overstating taxable Social Security. The operative 2026 computation uses the $18,150 standard deduction plus the $6,000 senior deduction."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model described the taxpayer as head of household even though no qualifying dependent exists and failed to apply the separate $6,000 senior deduction. The correct status is single with $24,150 of total deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's approximate $50,581 AGI overstates taxable Social Security and its result reflects only the ordinary aged standard deduction rather than the additional $6,000 senior deduction. Correct AGI is $43,637.65 and total deductions are $24,150."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a pre-TCJA 10%/15% schedule and personal-exemption framework instead of the operative 2026 10%/12% schedule and $6,000 senior deduction. It also overstated AGI by using too much taxable Social Security."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model improperly used qualifying-surviving-spouse status and its much larger standard deduction without a qualifying child. The filer is single, with $13,655.95 of taxable Social Security and $24,150 of total deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The submitted numeric value was $3,090 while its explanation stated a final liability of $3,584, violating the requirement that the explanation and value match. Its stated AGI also exceeds the correct $43,637.65 because it included too much taxable Social Security."
+us,scenario_042,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model mis-summed non-Social-Security income as $34,806 and derived $17,757 of taxable Social Security, then omitted the $6,000 senior deduction. The correct AGI is $43,637.65 and deductions total $24,150."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that age-related deductions and credits fully offset taxable income, but no nonrefundable credit applies and deductions total $24,150. The resulting ordinary taxable income is $18,559.65, not zero."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated unspecified deductions or nonrefundable credits as sufficient to erase the liability. Only $24,150 of deductions applies, no nonrefundable credit reduces the tax, and the ordinary taxable income produces $1,979.16."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model used a roughly $33,200 surviving-spouse deduction even though no qualifying child supports that filing status. The correct single-filer deductions total $24,150, and taxable Social Security is $13,655.95."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model overstated AGI to about $58,949 by including excessive taxable Social Security and then taxed qualified dividends and gains at 15%. Correct AGI is $43,637.65, and the $928 qualified dividends are excluded from ordinary taxable income."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied qualifying-surviving-spouse deductions without a qualifying child, reducing ordinary taxable income to only $6,036. The correct single-filer deductions are $24,150 and leave $18,559.65 subject to ordinary rates."
+us,scenario_042,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used a surviving-spouse standard deduction without the qualifying-child requirement. Applying single status, $43,637.65 of AGI, the $24,150 deductions, and the $928 qualified-dividend exclusion yields $1,979.16."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The answer implies that the model omitted the separate $6,000 senior deduction or used an excessive taxable Social Security amount, because its tax is near computations based only on the aged standard deduction. The correct ordinary taxable income after all deductions is $18,559.65."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a pre-TCJA standard-deduction, personal-exemption, and 10%/15% bracket regime. The operative computation uses the $18,150 standard deduction, the $6,000 senior deduction, and the 2026 10%/12% schedule."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed post-TCJA expiration, replacing the operative deduction structure with an $8,300 standard deduction and personal exemption and using 10%/15% brackets. The correct deductions are $18,150 plus $6,000, with ordinary tax computed under the 10%/12% schedule."
+us,scenario_042,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at $24,205, used a reverted pre-TCJA deduction of $10,233, and omitted the $6,000 senior deduction. It also failed to recognize the preferential treatment relevant to the full qualified-income amount used in the computation."
+us,scenario_042,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model double-counted the same $1,127 farm loss by treating farm income and farm-operations income as separate losses, and it omitted the $6,000 senior deduction. PolicyEngine deducts the farm loss once, uses AGI of $43,637.65, and applies $24,150 of deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_042,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model calculated taxable Social Security as $24,108.75 instead of $13,655.95, inflating AGI to $57,843.75. Even though it included the $6,000 senior deduction, the excessive Social Security inclusion left taxable income far too high."
+us,scenario_042,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model used married/qualifying-surviving-spouse Social Security thresholds and a $32,150 surviving-spouse deduction without a qualifying child. The correct computation uses single status, $13,655.95 of taxable Social Security, and $24,150 of deductions."
+us,scenario_042,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model incorrectly used MFJ-equivalent surviving-spouse thresholds, brackets, and a $39,850 deduction without a qualifying child. The correct single-filer deduction total is $24,150, leaving substantially more ordinary taxable income."
+us,scenario_042,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,The model incorrectly assigned qualifying-surviving-spouse status without a qualifying child and added the tax-exempt IRA distribution to Social Security provisional income. It also calculated taxable Social Security above the statutory 85% maximum and used an inflated surviving-spouse standard deduction.
+us,scenario_042,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model provided no derivation and returned a liability more than ten times the tax generated by the correct taxable income. The correct calculation applies ordinary rates to $18,559.65 after $24,150 of deductions and the $928 qualified-dividend exclusion, yielding $1,979.16."
 us,scenario_042,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_042,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that no employee wages were listed and that payroll tax should therefore be zero, but submitted $1,530 anyway. That amount contradicts its own payroll-tax-base reasoning and has no listed wage base supporting it."
 us,scenario_042,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model provided no payroll-tax output. It failed the required structured-output contract rather than completing the computation from the zero listed wage base.
 us,scenario_042,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented a roughly $24,000 retirement-income exclusion and simultaneously claimed taxable income was nearly eliminated while returning $953. The required calculation instead reaches $16,360.34 of taxable income after the actual Social Security subtraction, $12,671.36 standard deduction, and $950 exemptions, then subtracts the $300 property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated all $29,580 of Social Security benefits as Wisconsin taxable income, used an inapplicable estimated deduction, and then asserted that a $300–$400 property credit erased roughly $2,400–$2,600 of tax. Only the federally taxable Social Security amount is removed from federal AGI, yielding $16,360.34 of taxable income and $584.74 of tax before the $300 credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model started from an understated $33,806 AGI, inserted a $5,000 retirement subtraction, and mischaracterized the $700 and $250 exemptions as tax credits. Those amounts reduce taxable income, while the actual $300 property tax credit reduces $584.74 of tentative tax only to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted federally taxable Social Security from federal AGI, inserted a $5,000 retirement subtraction, and invoked unspecified real-estate and mortgage interactions to force the result to zero. The trace instead yields Wisconsin AGI of $29,981.71 and taxable income of $16,360.34, leaving $284.74 after the $300 credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an understated Wisconsin AGI near $33,735 and reduced the standard deduction to roughly $5,000 instead of applying the traced $13,655.95 subtraction and $12,671.36 standard deduction. This overstated taxable income and therefore the liability remaining after the school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated surviving-spouse status as married filing jointly, granted a $10,000 retirement subtraction, and used a roughly $22,010 joint standard deduction. It also treated the $700 personal and $250 age exemptions as dollar-for-dollar credits rather than reductions of taxable income, incorrectly eliminating the tax."
-us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from the federal-AGI starting point and invented an itemized-deduction credit large enough to erase the remaining liability. The applicable nonrefundable reduction is the $300 school property tax credit, leaving $284.74 after the traced deductions and exemptions."
-us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used $33,806 rather than $29,981.71 as Wisconsin AGI and omitted the $950 of personal and age exemptions. It also failed to subtract the $300 school property tax credit from the resulting tentative tax."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income as $29,446 instead of $16,360.34 by failing to apply the traced Wisconsin standard deduction and exemptions correctly. It also described a separate personal exemption credit even though the $700 and $250 exemptions reduce taxable income."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $1,137 implies that the model did not fully apply the $13,655.95 Wisconsin subtraction, $12,671.36 standard deduction, $950 exemptions, and $300 school property tax credit. Those steps reduce the requested liability to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model named the Social Security subtraction and standard deduction but computed them incorrectly or omitted the $950 exemptions and $300 school property tax credit. Correctly applying all four steps produces $16,360.34 of taxable income and $284.74 after the credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin taxable income as $25,235 rather than $16,360.34. Although it correctly subtracted the $300 school property tax credit, its incorrect standard-deduction and exemption calculation overstated tentative tax by $396.26."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $33,735 as Wisconsin AGI instead of the traced $29,981.71 and did not identify the $300 school property tax credit. Correct Wisconsin AGI, the $12,671.36 standard deduction, and $950 exemptions produce $584.74 before that credit and $284.74 afterward."
-us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model's reported answer was $1,348, while its own explanation ended with a net liability and submitted value of $1,617, violating the required agreement between explanation and numeric output. Neither figure reflects the traced $584.74 tentative tax less the $300 property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model falsely subtracted all IRA and private-pension income merely because the taxpayer was over 65, reducing Wisconsin AGI to $9,720. The traced Wisconsin subtraction is $13,655.95, primarily taxable Social Security, and leaves $16,360.34 taxable after the standard deduction and exemptions."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that pension exclusions and the standard deduction eliminated taxable income without calculating them. The actual deductions leave $16,360.34 taxable and $584.74 of tentative tax, of which only $300 is offset by the nonrefundable property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed without computation that deductions and credits reduced the liability to zero. The traced deductions leave $16,360.34 of taxable income, and the $300 nonrefundable credit does not fully offset the resulting $584.74 tax."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model incorrectly concluded that the school property tax credit fully offset tentative tax. The credit is $300, while tax after the Social Security subtraction, standard deduction, and exemptions is $584.74, leaving $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model explicitly failed to identify any nonrefundable state credit. The $5,010.88 of real-estate tax produces a $300 Wisconsin school property tax credit, which reduces $584.74 of tentative tax to $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invented a $24,000 retirement-income subtraction covering the IRA and pension distributions. The traced subtraction is $13,655.95, primarily federally taxable Social Security, and the remaining taxable income produces tax that the $300 property credit does not eliminate."
-us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model mentioned the Social Security subtraction and exemptions but omitted the $300 school property tax credit from its stated computation. The correct pre-credit tax is $584.74, and subtracting that credit yields $284.74."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model wrongly asserted that the standard deduction and retirement exclusions eliminated all taxable income. The traced deductions and exemptions leave $16,360.34 taxable, generating $584.74 before the $300 property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin AGI of $32,680 and a $10,683 standard deduction, rather than $29,981.71 and $12,671.36. It also overstated the school property tax credit as $366 instead of $300, so both its taxable-income base and credit computation were wrong."
-us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model started from $33,806 rather than the traced $29,981.71 Wisconsin AGI, omitted the $950 exemptions, and applied projected rates and deductions instead of the applicable parameters. It also stopped before subtracting the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model understated the federally taxable Social Security subtraction and therefore used Wisconsin AGI near $32,679 instead of $29,981.71; it also understated the standard deduction as $11,351. Although it applied the $300 credit, those base errors left taxable income about $4,018 too high."
-us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. It therefore failed the required output contract before any substantive tax calculation could be assessed.
-us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin AGI of $32,609.10 and a standard deduction near $12,023, rather than the traced $29,981.71 and $12,671.36. Its capital-gain adjustment and phase-down assumptions consequently overstated taxable income and tentative tax, even though it correctly applied a $300 property credit."
-us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a $24,000 retirement-income exclusion and a $24,000 surviving-spouse standard deduction, reducing taxable income to zero. The actual standard deduction is $12,671.36, and the traced adjustments leave $16,360.34 taxable."
-us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model's arithmetic is internally inconsistent: it stated roughly $7,200 of taxable income and $297 of pre-credit tax, then subtracted credits yet returned $417.88. It also treated exemptions and an itemized-deduction credit incorrectly; the applicable computation produces $584.74 before the $300 school property tax credit."
-us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The unexplained $1,830 submission implies that the model failed to apply the full Wisconsin Social Security subtraction, sliding standard deduction, exemptions, and school property tax credit. Those traced steps reduce the liability to $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented a roughly $24,000 retirement-income exclusion and then reported $953 despite reasoning that taxable income was nearly eliminated. It failed to use Wisconsin AGI of $29,981.71, the $12,671.36 standard deduction, $950 of exemptions, and the $300 property-tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model mischaracterized the $300 school property-tax credit as a large homestead credit capable of eliminating thousands of dollars of tax. The applicable nonrefundable credit is $300, reducing $584.74 to $284.74 rather than zero."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model treated the $700 and $250 Wisconsin exemptions as tax credits and added an unsupported itemized-deduction credit to erase the liability. Those amounts reduce taxable income, while the actual $300 property-tax credit reduces $584.74 of tax to $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model invented unspecified retirement, real-estate, and mortgage interactions that reduced a calculated positive liability to zero. Wisconsin instead applies $950 of exemptions as income reductions and a $300 property-tax credit, leaving $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an approximately $5,000 standard deduction and taxable income near $27,500 instead of the traced $12,671.36 deduction and $16,360.34 taxable income. This overstated bracket tax even after recognizing the property-tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated the surviving spouse as married filing jointly, assigned a roughly $21,151 standard deduction, and treated the $700 exemption as a credit. The applicable computation uses a $12,671.36 standard deduction and $950 of income exemptions, leaving $584.74 before the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model invoked an itemized-deduction credit large enough to eliminate about $790 of tax without calculating such a credit. The traced nonrefundable credit is only the $300 property-tax credit, leaving $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,The model omitted the $300 Wisconsin property-tax credit despite the listed real-estate taxes and also used the wrong Wisconsin AGI and rate parameters. The traced tax is $584.74 before that credit and $284.74 after it.
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $29,446 instead of $16,360.34, failing to apply the traced $12,671.36 standard deduction and $950 exemptions correctly. Its added $50 personal credit is not part of the traced calculation."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unsupported $1,137 answer implies excessive taxable income or omitted deductions and credits. Wisconsin taxable income is $16,360.34, producing $584.74 before the $300 property-tax credit and $284.74 afterward."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $860 estimate does not apply the complete traced deductions and credit. Wisconsin AGI of $29,981.71 is reduced by the $12,671.36 standard deduction and $950 exemptions before the resulting $584.74 tax is reduced by $300."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income as $25,235 rather than $16,360.34. Applying the correct standard deduction and exemptions yields $584.74 before the same $300 property-tax credit, not $981."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model started from $33,735 rather than the traced Wisconsin AGI of $29,981.71 and failed to identify the $300 property-tax credit. The complete calculation produces $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model supplied conflicting final figures—$1,348 in the answer and $1,617 in its explanation—and neither follows the traced calculation. Correct deductions and the $300 property-tax credit reduce the liability to $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted all IRA and private-pension income merely because the taxpayer is over 65. The traced Wisconsin subtractions total $13,655.95, leaving Wisconsin AGI of $29,981.71 rather than $9,720."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that pension exclusions and deductions eliminated taxable income, but the traced deductions leave $16,360.34 taxable. That amount generates $584.74 before the $300 credit, leaving $284.74."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The zero answer assumes unspecified retirement deductions and credits fully offset the liability. The actual deductions leave $16,360.34 taxable and the available $300 nonrefundable credit does not erase the $584.74 tax."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model correctly identified the school-property-tax credit but incorrectly concluded that it fully offsets tentative tax. The credit is $300 against $584.74, so $284.74 remains."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model failed to apply the $300 nonrefundable property-tax credit supported by $5,010.88 of real-estate taxes. It also did not reproduce the $16,360.34 taxable-income calculation that yields $584.74 before that credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied an erroneous $24,000 retirement-income subtraction to the IRA and pension. Wisconsin's traced subtractions total $13,655.95, leaving enough taxable income for $584.74 of tax before the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model stopped at an unsupported approximate tax and omitted the precise $300 property-tax-credit step. The correct pre-credit tax is $584.74 and the final amount is $284.74.
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed that retirement exclusions and the standard deduction eliminate all taxable income. The traced calculation leaves $16,360.34 taxable and $284.74 after the property-tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin AGI of $32,680 and a $10,683 standard deduction instead of $29,981.71 and $12,671.36. It also overstated the property-tax credit as $366 rather than the traced $300."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model's Wisconsin AGI of $32,680 and taxable income of $19,510 omit part of the traced Wisconsin subtractions and understate the standard deduction. Correct taxable income is $16,360.34, yielding $284.74 after the $300 credit."
+us,scenario_042,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model explicitly stopped before credits and therefore omitted the $300 Wisconsin property-tax credit. It also used projected brackets and a $33,806 starting amount instead of the traced Wisconsin AGI and deductions."
+us,scenario_042,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin AGI near $32,679 and a standard deduction near $11,351, producing taxable income about $4,018 too high. The traced values are $29,981.71 of Wisconsin AGI, a $12,671.36 deduction, and $16,360.34 of taxable income."
+us,scenario_042,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric value or explanation for the requested output, so its response could not be evaluated as a substantive computation."
+us,scenario_042,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated the taxable Social Security subtraction and used an incorrect phase-down formula, leaving Wisconsin AGI at $32,609.10 and taxable income at $19,636. The traced Wisconsin AGI is $29,981.71 and taxable income is $16,360.34."
+us,scenario_042,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a $24,000 retirement-income exclusion and a $24,000 surviving-spouse standard deduction. The traced standard deduction is $12,671.36, and the valid subtractions leave $16,360.34 taxable rather than zero."
+us,scenario_042,state_income_tax_before_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model treated all $20,506 of taxable income as subject to the bottom rate and invented an income-phased school levy credit of about $276. The traced progressive tax is $584.74 on $16,360.34, followed by a fixed $300 property-tax credit."
+us,scenario_042,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,The model's arithmetic is internally inconsistent: starting from roughly $297 of tax and subtracting stated credits cannot produce $417.88. It also invented personal-tax and itemized-deduction credits instead of applying the traced $300 property-tax credit to $584.74.
+us,scenario_042,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The unexplained $1,830 estimate implies that the model omitted major Wisconsin income subtractions, the sliding-scale standard deduction, exemptions, or the property-tax credit. Those steps yield $16,360.34 taxable income and a final liability of $284.74."
 us,scenario_042,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model understated Wisconsin homestead-credit household income by excluding substantial Social Security, taxable IRA distributions, pension income, dividends, interest, and capital gains, then applied an assumed 2024 maximum-credit formula instead of the 2026 eligibility computation. The complete computation produces no Wisconsin refundable credit."
 us,scenario_042,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric state_refundable_credits output or explanation, violating the required structured-output contract."
 us,scenario_043,federal_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model treated the age-66 head as eligible for the childless EITC and immediately applied the 7.65% phase-in rate to earned income. Under 2026 law, a taxpayer older than 64 fails the childless EITC age test, so the credit is $0."
@@ -2715,91 +2854,97 @@ us,scenario_043,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibi
 us,scenario_043,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_043,head_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_043,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_043,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model recognized that Colorado FAMLI belongs in payroll tax but submitted $315.14, a value unsupported by either its FICA-only calculation or its FICA-plus-FAMLI calculation. It also used an assumed 0.45% employee rate instead of the traced FAMLI computation yielding $17.99."
-us,scenario_043,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,The model incorrectly stated that Colorado has no mandatory state payroll tax and therefore omitted the $17.99 employee FAMLI contribution. It also rounded the incomplete federal FICA subtotal to a whole dollar.
-us,scenario_043,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model explicitly identified Colorado FAMLI and then excluded it as UI-like, contrary to the requested inclusion of mandatory employee state payroll taxes. That omission reduced the result to federal FICA alone."
-us,scenario_043,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,The model incorrectly asserted that no mandatory Colorado employee payroll tax applies. It omitted the $17.99 FAMLI contribution and returned only Social Security and Medicare taxes.
-us,scenario_043,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,The model calculated only the 7.65% federal FICA components and omitted Colorado's $17.99 employee FAMLI contribution. Its whole-dollar rounding further obscured the incomplete subtotal.
-us,scenario_043,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model incorrectly stated that Colorado has no mandatory employee-side state payroll tax. It omitted the $17.99 FAMLI contribution and returned federal FICA alone.
-us,scenario_043,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,The model incorrectly treated Colorado as having no applicable mandatory employee payroll contribution. Colorado FAMLI adds $17.99 to the Social Security and Medicare amounts.
-us,scenario_043,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the entire listed employer-sponsored insurance premium from FICA and FAMLI wages and forced the taxable wage base to zero. The traced employment wages remain subject to payroll tax, producing all three components totaling $330.73."
-us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,The model used only the 7.65% Social Security and Medicare rate. It omitted Colorado's mandatory employee FAMLI contribution of $17.99.
-us,scenario_043,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,The model summed only employee Social Security and Medicare taxes. It failed to add the $17.99 Colorado FAMLI employee contribution required in this payroll-tax output.
-us,scenario_043,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model incorrectly asserted that Colorado has no applicable employee state payroll tax and omitted the $17.99 FAMLI contribution. It also rounded its incomplete FICA calculation to $312.
-us,scenario_043,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the stated wages as outside the employee payroll-tax base without any supporting rule. Those wages generate Social Security, Medicare, and Colorado FAMLI taxes totaling $330.73."
-us,scenario_043,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,The model incorrectly excluded all mandatory Colorado employee payroll taxes. The missing Colorado FAMLI contribution is $17.99.
-us,scenario_043,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model stopped after applying the federal 7.65% Social Security and Medicare rate. It omitted the $17.99 Colorado FAMLI employee contribution.
-us,scenario_043,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model calculated and rounded only federal Social Security and Medicare taxes. It omitted Colorado's $17.99 mandatory employee FAMLI contribution.
-us,scenario_043,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,The model incorrectly concluded that Colorado has no employee payroll tax. Colorado FAMLI contributes $17.99 in addition to federal FICA.
-us,scenario_043,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model incorrectly stated that no mandatory state payroll tax applies and returned a rounded federal FICA subtotal. It omitted the $17.99 Colorado FAMLI employee contribution.
-us,scenario_043,payroll_tax,inkling,llm_error,payroll_tax_base,False,The model considered only employee Social Security and Medicare taxes. It omitted the mandatory Colorado FAMLI employee contribution of $17.99.
-us,scenario_043,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,The model incorrectly stated that Colorado has no mandatory employee state payroll tax. It therefore omitted the $17.99 FAMLI contribution and rounded the incomplete FICA subtotal.
-us,scenario_043,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,The model incorrectly asserted that Colorado has no mandatory employee state payroll tax and returned only federal FICA. The missing Colorado FAMLI contribution is $17.99.
-us,scenario_043,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,The model confused the absence of a general Colorado payroll income tax with the absence of mandatory employee payroll contributions. It omitted the $17.99 Colorado FAMLI contribution.
+us,scenario_043,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model used a 0.45% Colorado FAMLI employee rate instead of the applicable 2026 contribution that produces $17.99 on $4,088.15 of taxable wages. It then submitted $315.14 despite its own stated component calculation totaling $331.14, compounding the rate error with an output-transcription error."
+us,scenario_043,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory state payroll tax. It omitted the $17.99 employee Colorado FAMLI contribution and rounded the resulting federal-only FICA amount to a whole dollar.
+us,scenario_043,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model identified Colorado FAMLI but wrongly excluded it as an unemployment-insurance-like charge. FAMLI is a mandatory employee state payroll tax within the requested definition, adding $17.99 to federal FICA."
+us,scenario_043,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly asserted that no mandatory Colorado employee payroll tax applies. It therefore omitted the $17.99 Colorado FAMLI contribution.
+us,scenario_043,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model calculated only employee Social Security and Medicare taxes. It omitted the mandatory $17.99 Colorado FAMLI employee contribution from payroll tax.
+us,scenario_043,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory employee-side state payroll tax. Colorado FAMLI contributes $17.99 and raises the total from federal FICA alone to $330.73.
+us,scenario_043,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model treated federal FICA as the complete payroll-tax liability and incorrectly denied any applicable Colorado employee contribution. It omitted $17.99 of Colorado FAMLI tax.
+us,scenario_043,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted the listed employer-sponsored insurance premium from wages and reduced the FICA and FAMLI bases to zero. PolicyEngine's taxable wage calculation retains $4,088.15, generating $253.47 of Social Security tax, $59.28 of Medicare tax, and $17.99 of FAMLI tax."
+us,scenario_043,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model applied only the 7.65% federal FICA rate to gross wages. It omitted the mandatory $17.99 Colorado FAMLI employee contribution.
+us,scenario_043,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,The model summed only Social Security and Medicare taxes. It omitted the $17.99 Colorado FAMLI employee payroll tax.
+us,scenario_043,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,The model incorrectly stated that Colorado imposes no applicable employee payroll tax and calculated only federal FICA. It omitted the $17.99 FAMLI contribution and also rounded the federal amount down to $312.
+us,scenario_043,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model treated the reported employee wages as outside the payroll-tax base without identifying any exclusion. The $4,088.15 taxable wage base produces all three employee components totaling $330.73."
+us,scenario_043,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,The model incorrectly excluded mandatory Colorado employee payroll tax and returned federal FICA alone. The missing Colorado FAMLI component is $17.99.
+us,scenario_043,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,The model computed only the 7.65% federal Social Security and Medicare charge. It omitted the mandatory $17.99 Colorado FAMLI employee contribution.
+us,scenario_043,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model stopped after calculating federal FICA and rounded that subtotal to $313. It omitted the $17.99 Colorado FAMLI employee payroll tax.
+us,scenario_043,payroll_tax,grok-4.5,llm_error,state_local_rule,False,The model explicitly but incorrectly stated that Colorado has no employee payroll tax. Colorado FAMLI adds a mandatory $17.99 employee contribution to the federal FICA subtotal.
+us,scenario_043,payroll_tax,grok-4.6,llm_error,state_local_rule,False,The model correctly retained elective retirement contributions in the FICA base but incorrectly denied the Colorado payroll-tax component. It omitted $17.99 of employee FAMLI contributions.
+us,scenario_043,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model incorrectly stated that no mandatory state payroll tax applies and rounded federal FICA to $313. It omitted the $17.99 Colorado FAMLI contribution.
+us,scenario_043,payroll_tax,inkling,llm_error,state_local_rule,False,The model calculated the federal Social Security and Medicare components but did not apply Colorado FAMLI. The omitted mandatory employee contribution is $17.99.
+us,scenario_043,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory employee state payroll tax and returned rounded federal FICA alone. It omitted the $17.99 FAMLI contribution.
+us,scenario_043,payroll_tax,ox-alpha,llm_error,state_local_rule,False,The model limited payroll tax to Social Security and Medicare. It omitted the mandatory $17.99 Colorado FAMLI employee contribution.
+us,scenario_043,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,The model incorrectly stated that Colorado has no mandatory employee state payroll tax and therefore omitted $17.99 of FAMLI contributions. Its stated rounding from $312.74 to $312.73 is also arithmetically invalid.
+us,scenario_043,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,The model confused the absence of a separate Colorado payroll income tax with the absence of mandatory employee payroll contributions. Colorado FAMLI is included in this output and adds $17.99.
 us,scenario_043,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but used an obsolete $291 monthly maximum for all 12 months. The 2026 monthly schedule ranges from $298 to $304.68 and annualizes to $3,596.04."
-us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a $2,250 resource limit instead of the applicable elderly-household resource treatment and ignored categorical eligibility through TANF non-cash assistance. The household's $2,800 in liquid assets does not disqualify it."
-us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model derived zero net income and even identified a maximum near $300 per month, then replaced that result with an unsupported $200 monthly estimate. Zero net income produces the scheduled maximum allotments, totaling $3,596.04."
-us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model established that deductions drive net income to zero but arbitrarily used $192 per month instead of the maximum one-person allotment. The applicable monthly maxima total $3,596.04 over 2026."
-us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly found net income near zero but mislabeled $203 per month as close to the maximum. A zero-net-income one-person household receives the full 2026 monthly maxima, totaling $3,596.04."
-us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly calculated zero net income but held an outdated $292 maximum constant for the year. PolicyEngine applies the 2026 monthly amounts of $298–$304.68, which total $3,596.04."
-us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but used an excessive $328 monthly maximum. The applicable one-person maxima are $298–$304.68 across 2026 and total $3,596.04."
-us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the FY2025 $292 maximum throughout 2026 after correctly deriving negative net income. The 2026 monthly maximum schedule annualizes to $3,596.04."
-us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly reduced SNAP net income to zero but used $292 per month as the 2025–2026 maximum. The applicable 2026 amounts vary from $298 to $304.68 and total $3,596.04."
-us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly declared the household ineligible despite gross monthly income around $387, $2,800 of allowable liquid assets, and categorical eligibility through TANF non-cash assistance. Its zero answer also omits the deductions that reduce net income to zero and produce the maximum allotment."
-us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly found zero net income but used an obsolete $291 monthly maximum for every month. The applicable monthly maxima vary during 2026 and total $3,596.04."
-us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model recognized categorical eligibility and zero net income but rounded the annual maximum to $3,500 instead of applying the monthly benefit schedule. The scheduled 2026 allotments total $3,596.04."
-us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly derived zero net income but applied a constant $292 monthly maximum. The applicable 2026 monthly maxima range from $298 to $304.68 and annualize to $3,596.04."
-us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly concluded that net countable income is zero but used an outdated annual maximum of $3,504. Applying the 2026 monthly maximum schedule yields $3,596.04."
-us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"The model correctly identified maximum-allotment eligibility but used $302 per month for all 12 months. PolicyEngine applies varying monthly maxima of $298–$304.68, totaling $3,596.04."
-us,scenario_043,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model recognized eligibility but supplied an unsupported benefit estimate of $1,276 without applying the excess-medical deductions or maximum-allotment formula. Those deductions reduce net income to zero, and the 2026 monthly maxima total $3,596.04."
-us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the sparse facts as a lack of qualifying low-income indicators even though annual income is only about $4,648 and the elderly member receives categorical eligibility through TANF non-cash assistance. After allowable deductions, net income is zero and the household receives the maximum allotment."
-us,scenario_043,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly found zero net income but held the $298 monthly maximum constant for all 12 months. It omitted the within-year increase to as much as $304.68, so the monthly schedule totals $3,596.04 rather than $3,576."
-us,scenario_043,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model correctly established asset eligibility and zero net income but annualized $298 across the entire year. The maximum rises within 2026 to $304.68, producing $3,596.04 for the year."
-us,scenario_043,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly derived maximum-allotment eligibility but multiplied the initial $298 maximum by 12. Applying the within-year monthly changes through $304.68 yields $3,596.04."
-us,scenario_043,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model correctly reduced net income to zero but treated $298 as the maximum in every month. The benefit schedule changes within 2026 and totals $3,596.04."
-us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"The model incorrectly treated $2,800 of bank assets as disqualifying under a typical non-elderly resource limit. The age-66 household satisfies the applicable resource rules and is categorically eligible through TANF non-cash assistance."
-us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly found zero net income and maximum-allotment eligibility but used the obsolete $292 monthly amount. The applicable 2026 monthly maxima total $3,596.04."
-us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model rounded the maximum benefit to about $300 per month and multiplied that estimate by 12. The exact monthly schedule ranges from $298 to $304.68 and sums to $3,596.04."
-us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"The model correctly derived zero net income but substituted an approximate $300 monthly maximum for the exact scheduled amounts. The 2026 monthly maxima sum to $3,596.04, not $3,600."
-us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, violating the required structured-output contract."
-us,scenario_043,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly found zero net income but multiplied $298 by all 12 months. It omitted the within-year increase to $304.68, and the actual monthly schedule totals $3,596.04."
-us,scenario_043,snap,minimax-m3,llm_error,asset_resource,False,"The model asserted a zero result without applying the elderly household's applicable resource treatment or categorical eligibility through TANF non-cash assistance. The $2,800 bank balance does not bar eligibility, and deductions reduce net income to zero."
-us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly used an estimated $243 monthly maximum after finding very low net income and also mislabeled medical costs as a shelter deduction. The elderly medical deduction reduces net income to zero, triggering monthly maximum allotments of $298–$304.68 that total $3,596.04."
-us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model merely asserted that PolicyEngine produces no benefit and omitted the household's categorical eligibility, low income, allowable assets, and elderly medical deduction. Those rules reduce net income to zero and produce the maximum annual benefit of $3,596.04."
+us,scenario_043,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but used an outdated $291 monthly maximum. The applicable 2026 monthly maximums vary from $298 to $304.68 and total $3,596.04."
+us,scenario_043,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied a $2,250 general resource limit instead of the applicable elderly-household treatment and ignored categorical eligibility through TANF non-cash assistance. The $2,800 bank balance does not disqualify this household."
+us,scenario_043,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"After correctly deriving zero net income and identifying a roughly $300 maximum allotment, the model arbitrarily replaced that result with $200 per month. A zero-net-income one-person household receives the full monthly maximums, totaling $3,596.04."
+us,scenario_043,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly found near-zero net income but substituted $192 per month despite identifying a maximum near $292. Net income is zero, so no 30% reduction applies and the full 2026 monthly maximums total $3,596.04."
+us,scenario_043,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly found net income near zero but treated $203 as the maximum monthly allotment. The one-person maximums are $298–$304.68 during the year and sum to $3,596.04."
+us,scenario_043,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly computed zero net income but used a fixed $292 monthly maximum. PolicyEngine applies the 2026 monthly maximums of $298–$304.68 across their effective months, totaling $3,596.04."
+us,scenario_043,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but overstated the one-person maximum as $328 every month. The applicable monthly allotments are $298–$304.68, producing $3,596.04 annually."
+us,scenario_043,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used the FY2025 $292 maximum for all 12 months instead of the monthly values applicable in the 2026 tax year. The $298–$304.68 monthly allotments total $3,596.04."
+us,scenario_043,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly derived zero net income but used $292 per month as the 2025–2026 maximum. The tax-year calculation uses monthly maximums ranging from $298 to $304.68, totaling $3,596.04."
+us,scenario_043,snap,gemini-3.1-flash-lite-preview,llm_error,asset_resource,False,"The model incorrectly declared the low-income elderly household ineligible on income and assets. Gross and net income satisfy the limits, $2,800 in liquid assets is allowable, and TANF non-cash categorical eligibility applies."
+us,scenario_043,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly derived zero net income but used an outdated $291 monthly maximum. The applicable monthly maximums vary from $298 to $304.68 and total $3,596.04."
+us,scenario_043,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model rounded an unspecified estimated maximum to $3,500 instead of applying the exact monthly allotments. The $298–$304.68 monthly maximums sum to $3,596.04."
+us,scenario_043,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly computed zero net income but applied a fixed $292 monthly maximum. The 2026 monthly maximums range from $298 to $304.68 and total $3,596.04."
+us,scenario_043,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly concluded that the household receives the maximum allotment but assigned the wrong annual maximum of $3,504. Applying the monthly 2026 maximums yields $3,596.04."
+us,scenario_043,snap,glm-5.2,llm_error,thresholds_rates,False,"The model correctly found zero net income but effectively used $302 for every month. The maximum changes during the tax year from $298 to $304.68, and the month-specific amounts total $3,596.04."
+us,scenario_043,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model recognized eligibility but supplied an unsupported low estimate of $1,276. Zero net income produces the full one-person monthly maximums, totaling $3,596.04."
+us,scenario_043,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of generic low-income indicators as disqualifying despite the listed income, age, expenses, and assets. The household satisfies the income and resource rules and receives TANF non-cash categorical eligibility, producing a positive maximum allotment."
+us,scenario_043,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly found zero net income but held the $298 allotment constant for all 12 months. The monthly maximum rises as high as $304.68 during the year, so the month-specific total is $3,596.04."
+us,scenario_043,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model correctly established resource eligibility and zero net income but annualized $298 across the entire year. It omitted the months with higher maximum allotments up to $304.68, leaving the correct total at $3,596.04."
+us,scenario_043,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model used an estimated fixed $298 monthly maximum for the full year. PolicyEngine applies the month-specific maximums of $298–$304.68, which sum to $3,596.04."
+us,scenario_043,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model correctly reduced net income to zero but annualized the initial $298 maximum without the within-year increase. Applying each month's maximum produces $3,596.04."
+us,scenario_043,snap,grok-4.3,llm_error,asset_resource,False,"The model applied an unspecified typical asset limit and treated $2,800 as disqualifying. The household is elderly, its liquid assets are allowable, and TANF non-cash categorical eligibility also satisfies the resource pathway."
+us,scenario_043,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly found zero net income and resource eligibility but used a fixed $292 monthly maximum. The applicable monthly maximums are $298–$304.68 and total $3,596.04."
+us,scenario_043,snap,grok-4.6,llm_error,thresholds_rates,False,"The model correctly derived zero net income but selected the obsolete $292 one-person maximum. The 2026 month-specific maximums total $3,596.04."
+us,scenario_043,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model rounded the monthly maximum to $300 and multiplied that estimate by 12. The exact monthly values vary from $298 to $304.68 and sum to $3,596.04."
+us,scenario_043,snap,inkling,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but rounded the maximum to about $300 per month. The exact month-specific maximums yield $3,596.04 rather than $3,600."
+us,scenario_043,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output could not be parsed."
+us,scenario_043,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly found zero net income but multiplied the initial $298 maximum by all 12 months. The maximum rises to $304.68 during the year, and the month-specific allotments total $3,596.04."
+us,scenario_043,snap,minimax-m3,llm_error,asset_resource,False,"The model acknowledged the special elderly limits but then assigned zero without applying them. The $2,800 asset balance is allowable, income is far below the limits, and TANF non-cash categorical eligibility applies."
+us,scenario_043,snap,ox-alpha,llm_error,period_annualization,False,"The model correctly established eligibility and zero net income but held the $298 maximum constant all year. It omitted the within-year increase to as much as $304.68, so the correct annual total is $3,596.04."
+us,scenario_043,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly selected $243 as the one-person maximum after already finding very low income, and it conflated medical expenses with the shelter deduction. The elderly medical deduction reduces net income to zero, so the full $298–$304.68 monthly maximums apply and total $3,596.04."
+us,scenario_043,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that PolicyEngine produces no benefit without applying the stated SNAP pathways. The household passes the income and resource tests, qualifies categorically through TANF non-cash assistance, and receives the full maximum allotments because net income is zero."
 us,scenario_043,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model skipped the SSI income-limit eligibility gate and instead reduced the federal benefit rate by its own countable-income calculation. PolicyEngine applies the individual income test first; the person’s approximately $4,642 of income exceeds the limit, so the payable SSI amount is zero."
 us,scenario_043,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model treated SSI as a benefit-rate-minus-countable-income calculation despite the person failing the financial eligibility threshold. It then submitted $6,720 without deriving that figure from its own stated benefit rate and countable income, which produced approximately $9,360 rather than $6,720."
 us,scenario_043,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, so it failed the required submission contract."
-us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the EITC and child-related credits as the complete set of Colorado refundable credits. It omitted the Colorado sales tax refund, for which the age-66 filer qualifies and receives $19 at $4,625.40 of modified adjusted gross income."
-us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Colorado has no refundable individual credits. It omitted the refundable Colorado sales tax refund, which yields $19 for this qualifying senior filer."
-us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model substituted an estimated $1,130 TABOR refund for the applicable Colorado sales tax refund. The qualifying age-66 filer falls in the $19 sales-tax-refund amount based on modified adjusted gross income of $4,625.40."
-us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model limited its analysis to EITC-like and TABOR-related triggers and omitted the Colorado sales tax refund pathway. The filer's age and income generate a $19 sales tax refund.
-us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model invented a $225 senior housing or income-qualified credit instead of applying the Colorado sales tax refund schedule. The applicable senior sales tax refund is $19 at modified adjusted gross income of $4,625.40."
-us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The childless EITC age analysis does not resolve all Colorado refundable credits. The model omitted the separate Colorado sales tax refund, which awards this age-66 filer $19 based on modified adjusted gross income of $4,625.40."
-us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model conflated the applicable sales tax refund with estimated TABOR and senior housing credits and assigned an unsupported $1,200 amount. Applying the Colorado sales tax refund rules to this senior's $4,625.40 modified adjusted gross income yields $19."
-us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated the absence of a federal EITC as eliminating every Colorado refundable credit. It omitted the independent Colorado sales tax refund, which provides $19 to this qualifying age-66 filer."
-us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model stopped after rejecting the Colorado EITC and child-related credits. It failed to apply the Colorado sales tax refund, which produces $19 for this senior at the traced modified adjusted gross income."
-us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model incorrectly concluded that the household's income and composition satisfy no Colorado refundable-credit rules. Age 66 and the household's low modified adjusted gross income qualify it for a $19 Colorado sales tax refund.
-us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,The model considered only the property-tax/rent/heat rebate and treated the absence of qualifying expenses as dispositive. The Colorado sales tax refund is a separate pathway and supplies $19 based on the filer's age and modified adjusted gross income.
-us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model incorrectly made federal EITC eligibility a prerequisite for all Colorado refundable credits. The separate Colorado sales tax refund gives this qualifying age-66 filer $19.
-us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The zero answer omits the Colorado sales tax refund computation. The age-66 filer qualifies, and modified adjusted gross income of $4,625.40 yields a $19 refund."
-us,scenario_043,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,The model treated the lack of qualifying children and federal EITC as eliminating all Colorado refundable credits. It omitted the independent $19 Colorado sales tax refund available to this senior filer.
-us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"Although the model addressed the childless EITC age restriction, it incorrectly concluded that no other refundable credit applies. The Colorado sales tax refund separately awards the age-66 filer $19 at the traced income level."
-us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to recognize that the reported age and income trigger Colorado's sales tax refund. Applying that program produces a $19 refundable state credit.
-us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model focused on credits requiring earned income or qualifying children and omitted Colorado's senior sales tax refund pathway. The age-66 filer qualifies for a $19 refund based on modified adjusted gross income of $4,625.40."
-us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model catalogued child-based, earned-income, and property-related credits but omitted the Colorado sales tax refund. The listed age and income establish eligibility for a $19 sales tax refund without rent, property-tax, or heating-expense facts."
-us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model omitted the applicable Colorado sales tax refund rule. The age-66 filer qualifies, and the income-based schedule produces $19."
-us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,The model incorrectly limited refundable credits to the Colorado EITC and programs supported by dependents or listed expenses. Colorado's sales tax refund instead uses this filer's senior status and income and yields $19.
-us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model failed to apply the qualifying characteristics to Colorado's sales tax refund program. Age 66 and modified adjusted gross income of $4,625.40 produce a $19 refundable credit."
-us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The blanket conclusion that no refundable credit qualifies omits Colorado's sales tax refund. This senior filer qualifies for the program's $19 amount.
-us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model treated federal EITC, children, and other familiar circumstances as the only routes to a Colorado refund. It omitted the independent sales tax refund, which provides $19 to this age-66 low-income filer."
-us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model improperly awarded a $157 Colorado EITC by calculating 50% of a federal EITC despite the age-66 childless filer being outside the applicable federal EITC age range. It also omitted the actual $19 Colorado sales tax refund determined from senior eligibility and modified adjusted gross income.
-us,scenario_043,state_refundable_credits,inkling,llm_error,state_local_rule,False,The model correctly ruled out child-based and federal-EITC-linked credits but incorrectly stopped there. The separate Colorado sales tax refund gives the age-66 filer $19.
-us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,The model treated dependents and qualifying expenses as necessary for any Colorado refundable credit and overlooked the senior sales tax refund. The filer's age and income generate a $19 refund.
-us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model limited Colorado refundable credits to federal-credit derivatives and programs requiring a child, child care, or another listed expense. The Colorado sales tax refund requires none of those facts here and yields $19 from the filer's senior status and income."
-us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly concluded that no qualifying children or other factors meant no Colorado refund. Age 66 is the relevant additional factor for the sales tax refund, whose income schedule awards $19."
-us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model confined its analysis to the child tax credit and EITC and overlooked the Colorado sales tax refund. The age-66 filer qualifies for $19 based on modified adjusted gross income of $4,625.40."
-us,scenario_043,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,The model incorrectly tied the result to the absence of Colorado income-tax liability and failed to apply a refundable credit payable beyond liability. The Colorado sales tax refund independently produces $19 for this qualifying senior filer.
+us,scenario_043,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model treated the Colorado EITC and child-related credits as the complete set of refundable credits. It omitted the age- and employment-income-based Colorado sales tax refund, which yields $19 at $4,625.40 of modified AGI."
+us,scenario_043,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Colorado has no applicable refundable individual credits. It omitted the refundable Colorado sales tax refund available to this 66-year-old working filer, whose modified AGI produces $19."
+us,scenario_043,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model substituted an estimated $1,130 TABOR refund for the applicable Colorado sales tax refund and used an unsupported 2026 TABOR tier. The qualifying program in this computation awards $19 based on age, employment income, and $4,625.40 of modified AGI."
+us,scenario_043,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model incorrectly concluded that low wages and the absence of children prevent every Colorado refundable credit. The filer qualifies for the Colorado sales tax refund through age 66 and $4,072.71 of employment income, producing $19."
+us,scenario_043,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model invented a $225 senior housing or income-qualified credit without the required housing facts and failed to compute the Colorado sales tax refund. The applicable age-and-income pathway yields $19.
+us,scenario_043,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The federal and Colorado EITC age analysis does not exhaust Colorado refundable programs. The model omitted the Colorado sales tax refund, for which age 66, employment income, and $4,625.40 of modified AGI yield $19."
+us,scenario_043,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model combined speculative EITC, senior housing, and TABOR-style benefits into an unsupported $1,200 estimate. It failed to apply the actual Colorado sales tax refund schedule, which awards this filer $19."
+us,scenario_043,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model treated the absence of a federal EITC and qualifying children as eliminating all Colorado refundable credits. It omitted the Colorado sales tax refund, which independently yields $19."
+us,scenario_043,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model limited its analysis to the Colorado EITC and child-related credits. It omitted the independent Colorado sales tax refund for an age-66 filer with qualifying employment income, worth $19 here."
+us,scenario_043,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model incorrectly rejected refundable-credit eligibility based on income and household composition. Age 66 and $4,072.71 of employment income establish eligibility for the Colorado sales tax refund, and modified AGI sets it at $19."
+us,scenario_043,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model analyzed only the property-tax, rent, and heat PTC rebate and treated the absence of qualifying expenses as dispositive. It omitted the separate Colorado sales tax refund, which pays $19 without those expenses."
+us,scenario_043,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model incorrectly made federal EITC eligibility a prerequisite for every Colorado refundable credit. The Colorado sales tax refund has a separate age-and-income pathway that produces $19.
+us,scenario_043,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The zero answer omits the Colorado sales tax refund. Applying its age, employment-income, and modified-AGI rules produces $19."
+us,scenario_043,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,The model treated no qualifying children and zero federal EITC as eliminating all Colorado refundable credits. It omitted the separate Colorado sales tax refund worth $19.
+us,scenario_043,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model correctly excluded the childless EITC at age 66 but incorrectly stopped there. The Colorado sales tax refund separately covers this older working filer and yields $19.
+us,scenario_043,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The model failed to recognize that the reported age and employment income trigger the Colorado sales tax refund. The applicable modified-AGI bracket yields $19.
+us,scenario_043,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model generalized refundable credits as requiring qualifying children or a conventional earned-income-credit pathway. It omitted Colorado's sales tax refund for older working filers, which equals $19 here."
+us,scenario_043,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model exhaustively listed other credit categories but omitted the Colorado sales tax refund. That program uses the filer's age, employment income, and modified AGI and awards $19 without child or housing-expense facts."
+us,scenario_043,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model omitted the applicable Colorado sales tax refund. The age-66 filer has qualifying employment income, and the program's modified-AGI schedule produces $19."
+us,scenario_043,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model incorrectly required a federal EITC, dependents, or listed expenses for any Colorado refundable amount. The sales tax refund instead qualifies this older working filer and pays $19."
+us,scenario_043,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model failed to apply the Colorado sales tax refund eligibility pathway established by age 66 and employment income. The $4,625.40 modified AGI yields $19."
+us,scenario_043,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The blanket zero omits the Colorado sales tax refund. Its age-and-income calculation awards this household $19.
+us,scenario_043,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model treated the Colorado EITC, children, and unspecified other circumstances as the only routes to a refund. It omitted the sales tax refund pathway for an age-66 working filer, worth $19."
+us,scenario_043,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,The model correctly ruled out the Colorado EITC and the housing-dependent PTC rebate but failed to evaluate the separate sales tax refund. That refund requires neither children nor property-tax or rent facts and equals $19.
+us,scenario_043,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model wrongly assigned a federal and Colorado EITC to a childless filer age 66, producing the unsupported $157. It also omitted the distinct Colorado sales tax refund, whose age-and-income computation yields $19."
+us,scenario_043,state_refundable_credits,inkling,llm_error,state_local_rule,False,The model correctly found no child credit or Colorado EITC but incorrectly treated those as the only refundable programs. The Colorado sales tax refund independently provides $19.
+us,scenario_043,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model incorrectly treated no dependents or qualifying expenses as preventing all Colorado refundable credits. The sales tax refund uses age, employment income, and modified AGI instead, producing $19."
+us,scenario_043,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model limited Colorado refundable credits to federal-credit derivatives and child or childcare pathways. It omitted the independent Colorado sales tax refund, which equals $19 for this filer."
+us,scenario_043,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly concluded that low income and no children supply no qualifying factor. Age 66 combined with employment income qualifies the filer for Colorado's $19 sales tax refund.
+us,scenario_043,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,The model explicitly rejected a sales-tax refund after analyzing Colorado credits only as federal-credit derivatives and TABOR. It missed the separate Colorado sales tax refund program whose age-and-income schedule awards $19.
+us,scenario_043,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model focused on children and the Colorado EITC and incorrectly said no other qualifying characteristic existed. The head's age 66 and employment income are the qualifying characteristics for a $19 Colorado sales tax refund.
+us,scenario_043,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model incorrectly linked refundable-credit availability to having Colorado income-tax liability. The Colorado sales tax refund is refundable independently of that liability, and this filer's age and income produce $19."
 us,scenario_043,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_044,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the standard deduction directly from the full $56,806 Social Security benefit, effectively treating gross Social Security as taxable income. It failed to calculate taxable Social Security from provisional income before applying the joint-filer deduction, which leaves no taxable income."
 us,scenario_044,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,age_disability,False,"The model incorrectly described both spouses as older adults even though the spouse is 61, then produced tax without deriving the taxable portion of Social Security or taxable income. Only the 66-year-old head receives the age-based treatment, and the household's taxable Social Security remains below the applicable joint-filer deductions, producing zero tax."
@@ -2812,32 +2957,33 @@ us,scenario_044,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_erro
 us,scenario_044,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly asserted that Kansas taxes Social Security as ordinary income, then reduced the full $56,806 only by an $8,000 standard deduction and $4,500 of personal exemptions before applying tax brackets. Kansas instead subtracts the federally taxable Social Security benefits from Kansas adjusted gross income, so none of that benefit reaches the bracket calculation and the resulting tax is $0."
 us,scenario_044,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model classified the Kansas food sales tax credit as refundable and added $125 for each of two exemptions to state_refundable_credits. That credit is nonrefundable, so its asserted eligibility does not create any amount in this output."
 us,scenario_044,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly placed the $250 Kansas food sales tax credit in the refundable-credit total. Kansas treats this credit as nonrefundable, leaving state_refundable_credits at $0."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own computation yields approximately $2,173, but it submitted $2,540 without any supporting adjustment. It failed to carry its calculated tax into the value field."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $21,208 employer-sponsored insurance premium from the stated wages and used the wrong standard deduction. It then improperly used refundable EITC to reduce an output explicitly measured before refundable credits; the correct taxable income is $20,175.59 and the tax is $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly recalculated the liability as approximately $2,173 using the applicable deduction and brackets, then submitted $1,631. It failed to transfer its computed result to the final value."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model’s stated arithmetic produces $2,173, with no credits to reduce it, but it submitted $1,387. The final value contradicts every computation step in its explanation."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed approximately $2,173 and then replaced it with $2,029 under an unsupported claim of bracket rounding. Rounding the stated bracket calculation cannot produce a $144 reduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an obsolete estimated $15,350 standard deduction instead of the applicable $16,100 deduction. That overstated taxable income by about $750 and produced $2,273 rather than $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s own bracket calculation gives approximately $2,226, yet it submitted $1,130 without identifying any nonrefundable credit or other reduction. It discarded its computed liability instead of reporting it."
-us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $21,208 employer-sponsored insurance premium from the stated wages and applied post-sunset standard-deduction and personal-exemption assumptions. The applicable calculation keeps AGI at $36,275.59 and subtracts the $16,100 standard deduction, leaving $20,175.59 taxable."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the listed employer-sponsored insurance premium as a further subtraction from gross wages, reducing AGI to $15,068. AGI remains $36,275.59, so the $16,100 standard deduction leaves $20,175.59 taxable rather than zero."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model used head-of-household treatment even though this one-person household has no qualifying person and files single. It also failed to recognize that wages exceed the applicable $16,100 single standard deduction, leaving $20,175.59 taxable and $2,173.07 of tax."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA-style personal exemption and 15% bracket instead of the applicable 2026 $16,100 standard deduction and 10%/12% main-rate schedule. Those rules yield $20,175.59 of taxable income and $2,173.07 of tax."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $21,208 employer-sponsored insurance premium from wages and used an inapplicable standard-deduction-plus-personal-exemption regime. The correct AGI is $36,275.59 and taxable income after the $16,100 standard deduction is $20,175.59."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the employer-sponsored insurance premium and assumed a TCJA-expiration deduction and personal exemption. The applicable $16,100 standard deduction is taken from $36,275.59 of AGI, producing $20,175.59 of taxable income."
-us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied an inapplicable $8,300 standard deduction, personal exemption, and higher post-sunset rate structure. The applicable deduction is $16,100 and the main-rate calculation on $20,175.59 yields $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated parameters— a $15,400 deduction and a $12,250 first-bracket ceiling—instead of the applicable $16,100 deduction and 2026 rate thresholds. This overstated taxable income and produced $2,260.12 rather than $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction and unspecified nonrefundable credits as eliminating the liability. The $16,100 deduction leaves $20,175.59 taxable, and no facts support a nonrefundable credit, so tax remains $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented nonrefundable credits that fully offset the tax even though the household has no dependents or listed credit-qualifying expenses. The standard deduction leaves $20,175.59 taxable and no nonrefundable credit reduces the resulting $2,173.07 liability."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $15,750 standard deduction rather than the applicable $16,100 amount. Applying the correct deduction and exact main-rate thresholds to $20,175.59 of taxable income yields $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $21,208 employer-sponsored insurance premium from the stated annual wages. The wages produce AGI of $36,275.59, and the $16,100 standard deduction leaves $20,175.59 taxable rather than zero."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an approximate $15,000 standard deduction instead of $16,100 and did not apply the exact 2026 bracket thresholds. The correct taxable income is $20,175.59 and the resulting main-rate tax is $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the employer-sponsored insurance premium from wages and applied an inapplicable post-TCJA-sunset standard deduction and personal exemption. The applicable calculation uses $36,275.59 of AGI and a $16,100 standard deduction."
-us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA expiration and therefore used an inapplicable personal exemption, smaller standard deduction, and 15% second bracket. The applicable 2026 rules leave $20,175.59 taxable and tax it under the main 10% and 12% rates for $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. Its response therefore failed the required output contract.
-us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly found about $20,176 of taxable income but then used EITC and unspecified credits to erase the liability. EITC is refundable and is excluded from this before-refundable-credits output, while no listed facts generate a nonrefundable credit."
-us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model’s own bracket arithmetic produces $2,220.12 and identifies no nonrefundable credits, yet it submitted $424.76. It failed to report its computed liability; using the exact $16,100 deduction and applicable thresholds gives $2,173.07."
-us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that the standard deduction exceeds AGI even though $16,100 is less than $36,275.59, and its child-support discussion does not support reducing taxable income. The correct subtraction leaves $20,175.59 taxable, with no nonrefundable credit available to eliminate the $2,173.07 tax."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $2,173 from $20,176 of taxable income and the 10% and 12% brackets, then submitted $2,540 instead. Its numeric output contradicts its completed calculation."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $21,208 employer-sponsored insurance premium from the supplied gross wages, understated AGI, and used the wrong standard deduction. It also improperly used refundable EITC to reduce an output explicitly measured before refundable credits."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly recomputed the liability as approximately $2,173 using the $16,100 standard deduction and applicable brackets, but submitted $1,631. The final value does not follow from any calculation in its explanation."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly calculated $2,173 from taxable income of $20,176 and stated that no nonrefundable credits applied, then submitted $1,387. The unexplained final substitution discarded its correct computation."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model's stated deduction and bracket calculation produces approximately $2,173, not $2,029. Calling the $144 discrepancy bracket rounding is mathematically invalid because the supplied bracket inputs already yield the reference computation."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,350 estimated standard deduction instead of the applicable $16,100 deduction, overstating taxable income by $750. It also used outdated estimated bracket thresholds rather than the 2026 parameters that yield $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own estimated inputs produce $2,226 and it identified no applicable credit, yet it submitted $1,130. That final amount is an unsupported replacement of its computed tax liability."
+us,scenario_045,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $21,208 employer-sponsored insurance premium from gross wages and applied a postulated TCJA-expiration standard deduction and personal exemption. The applicable calculation retains $36,275.59 as AGI and deducts the $16,100 standard deduction, leaving $20,175.59 taxable."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the listed employer-sponsored insurance premium from gross wages, reducing AGI to $15,068. AGI is $36,275.59, so the $16,100 standard deduction leaves $20,175.59 taxable rather than zero."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model applied a head-of-household deduction even though the adult has no listed qualifying person and files single. It also failed to tax the income remaining after the applicable $16,100 single standard deduction."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied obsolete pre-TCJA assumptions, including a personal exemption and 15% second bracket. The 2026 computation instead uses the $16,100 standard deduction and the applicable 10% and 12% rates on $20,175.59 of taxable income."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced wages by the $21,208 employer-sponsored insurance premium and then used an inapplicable personal exemption and deduction total. The correct taxable income is $20,175.59 after subtracting only the $16,100 standard deduction from AGI."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the employer-sponsored insurance premium from gross wages and assumed TCJA expiration restored a personal exemption and smaller standard deduction. The applicable 2026 rules use $36,275.59 of AGI and a $16,100 standard deduction."
+us,scenario_045,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model substituted an $8,300 standard deduction plus a $5,300 personal exemption for the applicable $16,100 standard deduction. This overstated taxable income and produced tax under the wrong 2026 structure."
+us,scenario_045,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated 2026 parameters—a $15,400 standard deduction and $12,250 first-bracket ceiling—instead of the applicable $16,100 deduction and rate thresholds. Those parameter errors overstated taxable income and tax."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions and nonrefundable credits eliminate the liability without identifying any applicable credit. The $16,100 standard deduction leaves $20,175.59 taxable, and no listed fact generates a nonrefundable credit that offsets the resulting $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model invented nonrefundable credits sufficient to eliminate the tax despite no dependent, care, education, retirement, or other credit-generating fact. After the standard deduction, $20,175.59 remains taxable and no nonrefundable credit reduces the $2,173.07 liability."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,750 standard deduction instead of the applicable $16,100 amount, overstating taxable income by $350. Applying the actual deduction and 2026 brackets yields $2,173.07."
+us,scenario_045,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted the $21,208 employer-sponsored insurance premium from the supplied gross wages and concluded income fell below the standard deduction. The engine calculation uses $36,275.59 as AGI, leaving $20,175.59 taxable after the $16,100 deduction."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an approximate $15,000 standard deduction instead of $16,100, overstating taxable income by about $1,100. Its rounded bracket computation consequently exceeded the tax on the correct $20,175.59 taxable income."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced gross wages by the employer-sponsored insurance premium and applied a smaller postulated standard deduction plus a restored personal exemption. The applicable calculation uses unreduced AGI of $36,275.59 and the $16,100 standard deduction."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly retained wages in AGI but used an approximate $15,000 standard deduction rather than $16,100. It also used outdated bracket thresholds, causing tax to be overstated."
+us,scenario_045,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed TCJA expiration, replaced the applicable $16,100 standard deduction with an $8,000 deduction plus personal exemption, and taxed the second bracket at 15% instead of 12%. Those obsolete parameters overstated both taxable income and the marginal rate."
+us,scenario_045,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric value or explanation for the requested output. It therefore failed the required structured-output contract before any substantive tax calculation could be evaluated.
+us,scenario_045,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model correctly found approximately $20,176 of taxable income but then reduced the liability to zero using EITC and unspecified credits. EITC is refundable and is excluded from federal income tax before refundable credits, while no applicable nonrefundable credit is supported by the household facts."
+us,scenario_045,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own deduction and bracket arithmetic produces $2,220.12 and it states that no nonrefundable credits apply, yet it submitted $424.76. The final value is unsupported by and contradicts its stated computation."
+us,scenario_045,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model declared taxable income zero even while acknowledging that paid child support is not deductible. The $16,100 standard deduction does not exceed AGI of $36,275.59; it leaves $20,175.59 taxable, and no listed nonrefundable credit eliminates the tax."
 us,scenario_045,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly identified the childless EITC pathway and that its phaseout ends near $20,000, but then awarded $632 at $36,276. The childless EITC is fully phased out at that income, so this component is $0."
 us,scenario_045,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $21,208 of employer-sponsored insurance premiums from wages when computing EITC income, despite no stated pre-tax payroll treatment. It also produced $2,410 for a childless EITC whose maximum is far below that amount; using the full $36,276 leaves the credit fully phased out."
 us,scenario_045,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated earned income as $15,068 by subtracting the $21,208 insurance premium from $36,276 of gross wages. The EITC test uses the full $36,276 here, at which the childless credit is fully phased out."
@@ -2868,133 +3014,142 @@ us,scenario_045,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model
 us,scenario_045,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, violating the required output contract."
 us,scenario_045,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived approximately $2,249 of Social Security tax and $526 of Medicare tax and even stated a total of approximately $2,775, but submitted $2,837. Its final numeric output does not follow its own component calculation; the unrounded components total $2,775.08."
 us,scenario_045,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_045,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model ended eligibility under the gross-income screen and calculated no excess shelter deduction because it compared rent alone with half of adjusted income. Michigan categorical eligibility applies, and the applicable shelter calculation reduces net income to $1,102.06, producing the minimum allotment."
-us,scenario_045,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the full employer-sponsored insurance premium from SNAP income and then treated 30% of net income as the benefit rather than the expected contribution. SNAP instead subtracts that contribution from the maximum allotment and applies the minimum-allotment floor, yielding $287.68 annually."
-us,scenario_045,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the applicable shelter deduction, alternated between zero, minimum, and a large unsupported benefit, and never completed the statutory allotment formula. Correct deductions produce $1,102.06 of net income, after which the negative maximum-allotment calculation is replaced by the minimum allotment totaling $287.68."
-us,scenario_045,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model asserted that net income remained above the limit without completing the shelter-deduction calculation. The deductions reduce net income to $1,102.06, below the $1,304.17 limit, and the minimum allotment then applies."
-us,scenario_045,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated Michigan's 200% FPL gross-income threshold as dispositive. The household has categorical eligibility through TANF non-cash assistance, passes the resulting net-income test, and receives the minimum allotment."
-us,scenario_045,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test and stopped. Michigan categorical eligibility through TANF non-cash assistance requires continuation to the deduction and net-income calculations, which produce eligibility and the minimum allotment."
-us,scenario_045,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model calculated net income near $1,069—already below the applicable net-income limit—but then invoked 130% of poverty as a net threshold and rounded the benefit to zero. The benefit formula instead applies the one-person minimum-allotment floor, totaling $287.68 annually."
-us,scenario_045,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income screen as an absolute bar. It omitted Michigan categorical eligibility through TANF non-cash assistance and therefore never reached the deductions, passing net-income test, or minimum allotment."
-us,scenario_045,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated the 200% FPL gross-income screen as dispositive. Categorical eligibility through TANF non-cash assistance permits the household to proceed to the net-income test, which it passes before receiving the minimum allotment."
-us,scenario_045,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer implies that the model used an income threshold as an automatic disqualification. It omitted categorical eligibility and the deductions that reduce net income to $1,102.06, leading to the minimum allotment."
-us,scenario_045,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model recognized passage of the gross screen after child support but overstated net income after the earned-income, standard, and shelter deductions. Those deductions yield $1,102.06, below the net-income limit, and the household receives the minimum allotment."
-us,scenario_045,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model rejected the household solely from annual gross wages. It omitted Michigan categorical eligibility through TANF non-cash assistance, the passing net-income calculation, and the minimum-allotment floor."
-us,scenario_045,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model treated Michigan's 200% FPL gross threshold as an absolute eligibility limit. Categorical eligibility through TANF non-cash assistance allows the household to qualify after deductions and receive the minimum allotment.
-us,scenario_045,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model treated a negative maximum-allotment-minus-contribution result as zero. For an eligible one-person household, the minimum-allotment rule replaces that result with the statutory floor, totaling $287.68 annually."
-us,scenario_045,snap,glm-5.2,llm_error,categorical_eligibility,False,The model applied the ordinary 130% FPL gross-income screen and stopped. It failed to apply Michigan categorical eligibility through TANF non-cash assistance and therefore missed the passing net-income test and minimum allotment.
-us,scenario_045,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model's zero answer treats wages as leaving excessive SNAP resources without performing the applicable deductions. Those deductions produce $1,102.06 in net income, which passes the limit and triggers the minimum allotment."
-us,scenario_045,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model incorrectly claimed that qualifying status and household size were missing even though the prompt explicitly defines one household containing the head. It consequently defaulted SNAP to zero instead of applying categorical eligibility, the net-income calculation, and the minimum allotment."
-us,scenario_045,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the non-elderly, non-disabled gross-income limit as dispositive. Michigan categorical eligibility through TANF non-cash assistance overrides that shortcut, and the household passes the net-income test after deductions."
-us,scenario_045,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model used annual wages alone to deny SNAP. It omitted categorical eligibility and the deductions that reduce countable net income below the applicable limit, after which the minimum allotment applies."
-us,scenario_045,snap,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model treated the absence of a positive maximum-allotment-minus-contribution result as a zero benefit. Because the household is eligible after the stated deductions, the one-person minimum-allotment floor applies and totals $287.68."
-us,scenario_045,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model treated the one-person gross-income limit as an automatic bar. It omitted categorical eligibility through TANF non-cash assistance and never computed the passing net income or minimum allotment.
-us,scenario_045,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model overstated SNAP net income after deductions. The correct deduction sequence produces $1,102.06, below the $1,304.17 net-income limit, so the eligible household receives the minimum allotment."
-us,scenario_045,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test as dispositive. Michigan categorical eligibility through TANF non-cash assistance requires the net-income calculation, which the household passes before receiving the minimum allotment."
-us,scenario_045,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model relied on the ordinary 130% FPL test for a non-elderly, non-disabled adult. It omitted Michigan categorical eligibility through TANF non-cash assistance, which leads to a passing net-income test and the minimum allotment."
-us,scenario_045,snap,inkling,llm_error,categorical_eligibility,False,"The model treated the 130% FPL gross limit as insurmountable and asserted that deductions could not affect the result. Categorical eligibility prevents that gross-screen denial, and the applicable deductions reduce net income to $1,102.06."
-us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so the required numeric value could not be parsed."
-us,scenario_045,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated the 200% FPL broad-based gross screen as an absolute bar and excluded child support from the relevant eligibility computation. The engine applies categorical eligibility through TANF non-cash assistance and deductions that yield passing net income, followed by the minimum allotment."
-us,scenario_045,snap,minimax-m3,llm_error,categorical_eligibility,False,The model applied an approximate 130% FPL gross-income limit and stopped. It omitted Michigan categorical eligibility through TANF non-cash assistance and the subsequent net-income and minimum-allotment calculations.
-us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"The model's explanation concludes that SNAP equals zero, but it submitted $7,032, directly violating the required agreement between explanation and numeric output. It also applied the ordinary 130% FPL gross screen instead of categorical eligibility and the minimum-allotment calculation."
-us,scenario_045,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that net income exceeded the limit without applying the full deduction sequence. Correct deductions produce $1,102.06 in net income, below the $1,304.17 limit, and the minimum allotment applies."
+us,scenario_045,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model denied eligibility under gross and net income screens without applying categorical eligibility through TANF non-cash assistance. Its deduction calculation also left net income at $1,776 instead of the traced $1,102.06, omitting the shelter computation that produces eligibility and the minimum allotment."
+us,scenario_045,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the entire $21,208 employer-sponsored insurance premium from SNAP income and then invented a $258 monthly benefit. The correct SNAP deductions produce $1,102.06 monthly net income, and $298 minus the $330.60 expected contribution is floored at the minimum allotment rather than yielding $258."
+us,scenario_045,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used inconsistent deduction and shelter calculations, first deriving no positive allotment and then replacing that result with an unsupported $2,718 estimate. The traced deductions yield $1,102.06 monthly net income, so the negative formula result is replaced by the minimum allotment, not a large shelter-driven benefit."
+us,scenario_045,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated its estimated net-income excess as disqualifying and never applied the categorical-eligibility pathway and complete deduction calculation. Monthly net income is $1,102.06, below the $1,304.17 limit, and the negative benefit formula is subject to the minimum-allotment floor."
+us,scenario_045,snap,claude-opus-5,llm_error,categorical_eligibility,False,"The model denied SNAP solely because annual wages exceeded an estimated 200% FPL gross limit. It missed categorical eligibility through TANF non-cash assistance and therefore never calculated the qualifying $1,102.06 monthly net income or minimum allotment."
+us,scenario_045,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income test as an absolute bar. Michigan categorical eligibility through TANF non-cash assistance prevents that shortcut, after which the household passes the net-income test and receives the minimum allotment."
+us,scenario_045,snap,claude-sonnet-5,llm_error,other,False,"The model's own tentative calculation placed net income below the applicable limit, but it then declared the household ineligible using an irrelevant 130% comparison and rounded the benefit to zero. For an eligible one-person household, a negative maximum-allotment-minus-contribution result is floored at the SNAP minimum allotment, not zero."
+us,scenario_045,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used the ordinary 130% FPL gross-income screen as dispositive. It omitted categorical eligibility through TANF non-cash assistance, the subsequent net-income test, and the minimum-allotment floor."
+us,scenario_045,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model denied the household under an estimated 200% FPL gross-income limit. It failed to apply categorical eligibility through TANF non-cash assistance and therefore omitted the qualifying net-income calculation and minimum allotment.
+us,scenario_045,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer implies that the model treated income as categorically disqualifying. The household instead qualifies through TANF non-cash categorical eligibility, passes the $1,304.17 net-income limit with $1,102.06, and receives the minimum allotment."
+us,scenario_045,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model accepted that child support brought income through the gross screen but miscomputed the earned-income, standard, and shelter deductions. Those deductions produce $1,102.06 monthly net income, below the $1,304.17 limit, followed by the minimum-allotment floor."
+us,scenario_045,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model treated gross annual income as an automatic disqualifier. It omitted TANF non-cash categorical eligibility and the net-income calculation that establishes eligibility and triggers the minimum allotment.
+us,scenario_045,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model treated Michigan's 200% FPL gross threshold as an absolute bar. It missed categorical eligibility through TANF non-cash assistance and never reached the passing net-income test or minimum-allotment calculation.
+us,scenario_045,snap,gemini-3.7-flash,llm_error,other,False,"The model conflated a negative maximum-allotment-minus-expected-contribution calculation with a zero award. Once the household passes eligibility with $1,102.06 net income, that negative result is floored at the minimum allotment."
+us,scenario_045,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% FPL gross-income screen and stopped. It omitted Michigan categorical eligibility through TANF non-cash assistance, under which the household proceeds to and passes the net-income test."
+us,scenario_045,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model's zero assumes wage income precludes SNAP without applying the household's categorical-eligibility pathway and statutory deductions. The resulting $1,102.06 monthly net income passes the test and produces the minimum allotment."
+us,scenario_045,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The model incorrectly claimed that qualifying status and household size were missing even though the prompt specifies one head and one household group and directs unlisted facts to be false or zero. Using that specified one-person unit, the household qualifies and receives the minimum SNAP allotment."
+us,scenario_045,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model treated the non-elderly gross-income limit as dispositive. It omitted categorical eligibility through TANF non-cash assistance and therefore failed to calculate the passing net income and minimum allotment.
+us,scenario_045,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model denied eligibility solely from wages exceeding an unspecified Michigan income threshold. It missed categorical eligibility through TANF non-cash assistance and the deductions that reduce monthly net income to $1,102.06."
+us,scenario_045,snap,gpt-5.6-sol,llm_error,other,False,The model treated the absence of a positive maximum-allotment-minus-contribution result as a zero benefit. The eligible one-person household receives the minimum allotment when its $330.60 expected contribution exceeds the $298 maximum allotment.
+us,scenario_045,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the one-person income limit as an absolute gross-income cutoff. It omitted TANF non-cash categorical eligibility, the passing net-income test, and the minimum-allotment floor."
+us,scenario_045,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that net income exceeded the limit. The complete SNAP deduction calculation yields $1,102.06 monthly net income, below the $1,304.17 threshold, after which the minimum allotment applies."
+us,scenario_045,snap,grok-4.5,llm_error,categorical_eligibility,False,The model stopped at the ordinary 130% FPL gross-income test. It failed to apply categorical eligibility through TANF non-cash assistance and the subsequent passing net-income and minimum-allotment calculations.
+us,scenario_045,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model treated both the 130% and 200% FPL gross screens as dispositive and relegated child support solely to a later deduction. It missed categorical eligibility through TANF non-cash assistance, which allows the household to reach the net-income test and qualify for the minimum allotment."
+us,scenario_045,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model applied the standard 130% FPL gross test based on the head being non-elderly and non-disabled. It omitted the separate TANF non-cash categorical-eligibility pathway and consequently never computed the passing net income or minimum allotment.
+us,scenario_045,snap,inkling,llm_error,categorical_eligibility,False,"The model treated the 130% FPL gross limit as insurmountable and asserted that deductions could not overcome it. It omitted categorical eligibility through TANF non-cash assistance and the deductions that produce qualifying net income of $1,102.06."
+us,scenario_045,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric SNAP output or explanation, so the required output could not be parsed or evaluated."
+us,scenario_045,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated the 200% FPL gross screen as the highest available eligibility route and stopped before deductions. It omitted categorical eligibility through TANF non-cash assistance, under which the household passes the net-income test and receives the minimum allotment."
+us,scenario_045,snap,minimax-m3,llm_error,categorical_eligibility,False,The model applied the ordinary 130% FPL gross-income limit as an automatic denial. It missed TANF non-cash categorical eligibility and the subsequent net-income and minimum-allotment computations.
+us,scenario_045,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model stopped at the 130% FPL gross-income screen. It omitted categorical eligibility through TANF non-cash assistance, which leads to a passing $1,102.06 net-income result and the minimum allotment."
+us,scenario_045,snap,qwen-3.7-max,llm_error,other,False,"The model's submitted $7,032 contradicts its own explanation that the household receives zero and exceeds the $298 monthly maximum allotment even before the expected contribution. It neither applied the categorical-eligibility and minimum-allotment derivation nor transferred its stated conclusion into the numeric field."
+us,scenario_045,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that net income exceeded the eligibility threshold. The traced deductions yield $1,102.06 monthly net income against a $1,304.17 limit, and the eligible household then receives the minimum allotment."
 us,scenario_045,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model never used Michigan's $5,950 personal exemption and instead cycled through unsupported exemption estimates before submitting $1,229, a value not produced by its own calculations. The required computation is $30,325.59 times 4.25%, yielding $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model wrongly deducted the separately listed $21,208 employer-sponsored insurance premiums from wages, imported a federal standard deduction into Michigan taxable income, and treated the refundable Homestead Property Tax Credit as nonrefundable. Michigan instead taxes $36,275.59 less the $5,950 personal exemption, producing $1,288.84 before refundable credits."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model used an estimated $5,800 personal exemption instead of Michigan's 2026 exemption of $5,950. That overstated taxable income by about $150 and produced $1,295 instead of applying 4.25% to $30,325.59."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's submitted $1,024 contradicts its own completed calculation of $1,295 and has no supporting computation. It also used a $5,800 exemption rather than $5,950; the required calculation yields $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model used an estimated $5,800 exemption and then subtracted an unidentified nonrefundable credit despite identifying none. No applicable nonrefundable credit reduces the tax, so 4.25% applies to $30,325.59 for $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied an inapplicable 4.05% rate instead of Michigan's 4.25% rate and used a rounded $6,000 exemption instead of $5,950. Applying 4.25% to $36,275.59 minus $5,950 yields $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model computed approximately $1,295 from its assumed inputs and then arbitrarily increased the submitted value to $1,490 through a nonexistent rounding adjustment. It also used a $5,800 exemption instead of $5,950; the prescribed calculation yields $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced federal AGI to $15,068 by subtracting the separately listed employer-sponsored insurance premiums from annual wages. PolicyEngine's AGI is $36,275.59, and subtracting only Michigan's $5,950 personal exemption leaves $30,325.59 taxable."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 as AGI by deducting the employer-sponsored insurance premiums from the reported annual wages, then used an outdated $5,600 exemption. The applicable AGI is $36,275.59 and the exemption is $5,950, producing $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The submitted $1,150 does not result from Michigan's stated 4.25% rate applied after the 2026 personal exemption. The correct base is $36,275.59 minus $5,950, and 4.25% of that base is $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model used an implied $5,800 personal exemption rather than Michigan's 2026 amount of $5,950. Applying 4.25% to the resulting $30,325.59 taxable income gives $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI to $15,068 by subtracting employer-sponsored insurance premiums from the listed wages and then used a $5,600 exemption. Michigan taxable income is $30,325.59 after the $5,950 personal exemption, yielding $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $15,068 as AGI after deducting the employer-sponsored premiums and used a $5,600 personal exemption. The calculation starts from $36,275.59 and subtracts the $5,950 Michigan exemption, producing $1,288.84 of tax."
-us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model used a $5,800 personal exemption instead of the 2026 Michigan exemption of $5,950. This overstated taxable income by about $150; 4.25% of the correct $30,325.59 base is $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model invented a $20,000 Michigan standard deduction for this filer and treated the Homestead Property Tax Credit as a nonrefundable offset, then submitted zero despite its own $121 calculation. Michigan uses the $5,950 personal exemption here, while the Homestead credit belongs outside tax before refundable credits, leaving $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly assumed Michigan's deduction structure eliminated all taxable income. Only the $5,950 personal exemption reduces the $36,275.59 AGI here, leaving $30,325.59 taxed at 4.25%."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model assumed unspecified deductions and nonrefundable credits fully offset Michigan tax. No such offset applies: the $5,950 exemption leaves $30,325.59 taxable and $1,288.84 due before refundable credits."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model estimated the personal exemption at $5,900 instead of using Michigan's 2026 amount of $5,950 and rounded AGI to $36,276. Using traced AGI of $36,275.59 and the correct exemption gives taxable income of $30,325.59 and tax of $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied a 4.05% Michigan rate instead of 4.25% and used a $5,800 exemption instead of $5,950. The correct rate applied to $30,325.59 yields $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced federal AGI to $15,068 by deducting employer-sponsored insurance premiums from the listed wages, then estimated the exemption at $5,900. Michigan starts from $36,275.59 and subtracts the $5,950 exemption, yielding $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model used an assumed $5,900 personal exemption rather than the 2026 Michigan amount of $5,950 and rounded AGI. The traced taxable income is $30,325.59, whose 4.25% tax is $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model incorrectly asserted that Michigan deductions and exemptions offset all $36,276 of income. The applicable $5,950 exemption leaves $30,325.59 taxable, so the liability is $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly used $15,068 of federal AGI by deducting the employer-sponsored premiums from reported wages and also used a projected $5,880 exemption. The correct AGI and exemption are $36,275.59 and $5,950, respectively, producing $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model used a $5,000 Michigan personal exemption rather than the 2026 amount of $5,950. This overstated taxable income by $950; the correct $30,325.59 base taxed at 4.25% produces $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model subtracted a Homestead Property Tax Credit from this before-refundable-credits output. That credit is refundable and is reported separately, so it does not reduce the $1,288.84 tax calculated from Michigan taxable income."
-us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured result was missing."
-us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used a $6,000 personal exemption instead of Michigan's $5,950 exemption and rounded AGI to $36,276. The traced $30,325.59 taxable base produces $1,288.84 at 4.25%."
-us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model calculated approximately $1,329, correctly recognized that the Homestead Property Tax Credit is excluded from this output, and then inexplicably submitted zero. It also used a $5,000 exemption rather than $5,950; the correct liability is $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model calculated $1,303.73 from its assumed inputs but submitted $1,116.01 without any computation supporting that value. It also used a $5,600 exemption instead of $5,950; applying 4.25% to $30,325.59 yields $1,288.84."
-us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted both a purported Michigan standard deduction and the $5,200 child-support payment to reduce taxable income to zero. Child support is not deductible, and this filer instead subtracts only the $5,950 personal exemption from $36,275.59, leaving $1,288.84 of tax."
-us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly restricted the Michigan Homestead Property Tax Credit to homeowners. Renters qualify through deemed property tax equal to 23% of rent, producing a $760.79 refundable credit here."
-us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly derived the Homestead Property Tax Credit as approximately $760.78, then submitted $884 without any supporting computation. Its final value contradicts its own correct calculation."
-us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model focused on the Michigan EITC and Home Heating Credit but omitted the refundable Homestead Property Tax Credit for renters. Applying 23% of the listed rent and the household-resource exemption yields $760.79, not $1,209."
-us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model treated the zero Michigan EITC as exhausting state refundable credits and omitted the Homestead Property Tax Credit. That refundable renter credit equals $760.79.
-us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model read the absence of separately listed property tax as disqualifying, despite the listed rent. Michigan deems 23% of rent to be property tax for the Homestead Property Tax Credit, yielding $760.79."
-us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model acknowledged that the childless federal and Michigan EITCs are zero, then invented a positive EITC-based amount without applying the renter Homestead Property Tax Credit. The actual refundable amount comes entirely from that credit and equals $760.79."
-us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced total household resources to $15,068 and also used 20% rather than 23% of rent, creating an overstated Homestead credit and a spurious EITC. Household resources are $36,275.59 and countable property tax is $2,428.80, yielding $760.79."
-us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly used 23% of rent but incorrectly reduced total household resources to $15,068. Using $36,275.59 in household resources produces the $760.79 Homestead Property Tax Credit."
-us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the renter pathway to Michigan's Homestead Property Tax Credit. The listed $10,560 rent supports a refundable credit of $760.79."
-us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model equated a zero federal EITC with zero Michigan refundable credits. It failed to calculate the independent Homestead Property Tax Credit for renters, which is $760.79."
-us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model used 20% rather than 23% of rent, introduced an inapplicable positive EITC, and did not use the traced household-resource calculation. The Homestead Property Tax Credit alone equals $760.79."
-us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model assigned a positive federal EITC and Michigan match even though a childless filer with $36,276 of earnings is beyond the EITC phaseout. It also omitted the $760.79 renter Homestead Property Tax Credit."
-us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model identified the correct credit but miscomputed it. Countable property tax of $2,428.80 and the household-resource exemption produce $760.79, not $695.48."
-us,scenario_045,state_refundable_credits,glm-5.2,llm_error,other,False,"The model used the wrong 20% renter conversion and calculated $570.70, but then submitted zero despite stating that the household qualified. Michigan uses 23% of rent here, and the resulting refundable credit is $760.79."
-us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to identify the Michigan Homestead Property Tax Credit supported by the listed rent and household resources. The renter calculation yields $760.79.
-us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly treated a dependent or other unlisted condition as necessary for all Michigan refundable credits. A childless renter can claim the Homestead Property Tax Credit, which equals $760.79 here."
-us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model used 20% instead of Michigan's applicable 23% conversion of rent to countable property tax. The correct countable property tax is $2,428.80, and the credit is $760.79."
-us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model improperly reduced household resources to $15,068 and added a positive state EITC. With household resources of $36,275.59, the Homestead Property Tax Credit is $760.79 and the EITC component is zero."
-us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model incorrectly treated being childless as disqualifying the household from every Michigan refundable credit. The Homestead Property Tax Credit does not require a child and provides $760.79.
-us,scenario_045,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted the renter Homestead Property Tax Credit despite the listed rent and qualifying household resources. That credit produces $760.79.
-us,scenario_045,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used $15,068 instead of $36,275.59 for household resources, applied a 3.5% threshold rather than the traced exemption calculation, and added a spurious EITC. The correct Homestead credit is $760.79 with no EITC amount."
-us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model failed to apply Michigan's refundable Homestead Property Tax Credit to the renter. The supplied rent and resources yield $760.79.
-us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,The model incorrectly conditioned the Homestead Property Tax Credit's refundable amount on exceeding state income-tax liability. The credit is refundable independently of that liability and equals $760.79.
-us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits.
-us,scenario_045,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,The model used 20% instead of 23% of rent and a 3.5% resource percentage rather than the applicable household-resource exemption calculation. The correct refundable Homestead credit is $760.79.
-us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model wrongly disregarded the explicitly listed pre-subsidy rent and concluded that no rent was paid. That annual rent supplies the renter property-tax equivalent used to calculate a $760.79 Homestead credit.
-us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model asserted that the listed facts did not establish Homestead Property Tax Credit qualification, overlooking the rent and household resources that do so. The resulting refundable credit is $760.79."
-us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model omitted Michigan's refundable Homestead Property Tax Credit for qualifying renters. Applying it to the listed rent and resources yields $760.79.
+us,scenario_045,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model never used the $5,950 Michigan personal exemption and ultimately submitted $1,229 despite its own calculations clustering around $1,272–$1,291. The required base is $30,325.59, producing $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted both the $21,208 employer-sponsored insurance input and a federal standard deduction from wages when calculating Michigan taxable income. Michigan instead starts from AGI of $36,275.59 and subtracts the $5,950 personal exemption, while the Homestead Property Tax Credit does not erase this before-refundable-credit liability."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $5,800 personal exemption instead of Michigan’s $5,950 exemption. That overstated taxable income by about $150 and produced $1,295 rather than $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly described a $1,295 calculation using its assumed exemption but submitted $1,024, a number unsupported by its reasoning. It also used $5,800 rather than the applicable $5,950 exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an estimated $5,800 exemption and then invented an unspecified $60 reduction for nonrefundable credits. No such reduction applies; the $5,950 exemption yields $30,325.59 of taxable income and $1,288.84 of tax."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a 4.05% rate and a projected $6,000 exemption. The applicable computation uses the Michigan rate producing $1,288.84 on $30,325.59 after the $5,950 exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated approximately $1,295 from its assumed $5,800 exemption, then submitted $1,490 without any computational basis. Using the $5,950 exemption produces $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced the $36,276 wage input by the separately listed $21,208 employer-sponsored insurance premiums to create federal AGI of $15,068. The trace uses AGI of $36,275.59, less the $5,950 Michigan exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $15,068 as federal AGI by subtracting the employer-premium input from wages, then used an outdated $5,600 exemption. Michigan taxable income is $30,325.59 after the $5,950 exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $1,150 implies an excessive deduction or an incorrect effective rate. Michigan’s $5,950 personal exemption leaves $30,325.59 taxable and yields $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $5,800 personal exemption instead of $5,950. This overstated the Michigan taxable base and tax."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced wages by the $21,208 employer-premium input to obtain AGI of $15,068 and also used a $5,600 exemption. The correct Michigan base is $36,275.59 minus $5,950."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $15,068 as AGI, reflecting subtraction of the separately listed employer premiums from annual wages. The engine uses $36,275.59 of AGI and the $5,950 Michigan exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $5,800 exemption rather than the applicable $5,950 exemption. Michigan taxable income is therefore $30,325.59, not $30,476."
+us,scenario_045,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a $20,000 Michigan standard deduction for this 44-year-old filer and treated the Homestead Property Tax Credit as a nonrefundable offset to this output. Michigan instead subtracts only the $5,950 personal exemption here, and the submitted zero also contradicts its own $121 calculation."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that Michigan deductions eliminated all taxable income. Only the $5,950 personal exemption applies in the traced calculation, leaving $30,325.59 taxable."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified standard deductions and nonrefundable credits fully offset Michigan tax. The applicable personal exemption is $5,950, and no traced nonrefundable credit reduces the resulting $1,288.84 liability."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated the personal exemption at $5,900 rather than using $5,950. It therefore overstated taxable income and tax by $2.14."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model applied a 4.05% rate and a $5,800 exemption. The traced Michigan calculation uses the $5,950 exemption and produces $1,288.84 on $30,325.59."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly set federal AGI to $15,068 by netting the employer-premium input against wages. Michigan starts from $36,275.59 of AGI and subtracts the $5,950 exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model assumed a $5,900 personal exemption rather than the applicable $5,950. The resulting $50 overstatement of taxable income caused its $2.14 tax overstatement."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model invented sufficient Michigan standard deductions and exemptions to offset all $36,276 of income. The traced calculation subtracts only the $5,950 personal exemption and leaves $30,325.59 taxable."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly used $15,068 as federal AGI after subtracting employer premiums from the stated annual wages. The correct starting AGI is $36,275.59, followed by the $5,950 Michigan exemption."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a projected $5,800 exemption instead of the applicable $5,950 exemption. This made its taxable income $150 too high."
+us,scenario_045,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $5,000 Michigan personal exemption instead of $5,950. It consequently overstated taxable income by about $950 and tax by about $40."
+us,scenario_045,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model subtracted an estimated Homestead Property Tax Credit from the requested tax-before-refundable-credits output. That refundable credit belongs in the separate state_refundable_credits quantity and does not reduce the $1,288.84 reported here."
+us,scenario_045,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_045,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $6,000 exemption instead of $5,950. It also misstated the arithmetic: $30,276 multiplied by 4.25% is $1,286.73, while the traced $30,325.59 base yields $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model computed approximately $1,329, correctly noted that the Homestead Property Tax Credit is excluded from this output, and then submitted zero without a valid computation. It also used a $5,000 exemption instead of $5,950."
+us,scenario_045,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used a $5,000 personal exemption rather than Michigan’s $5,950 exemption. This overstated the taxable base by about $950 and produced $1,329.23 instead of $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model calculated $1,303.73 using an incorrect $5,600 exemption but then submitted $1,116.01 without any supporting step. The applicable $5,950 exemption leaves $30,325.59 taxable and produces $1,288.84."
+us,scenario_045,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted child support and an unspecified standard deduction to reduce Michigan taxable income to zero. Child support is not deductible, and the traced calculation subtracts the $5,950 personal exemption from $36,275.59 of AGI."
+us,scenario_045,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly restricted the Homestead Property Tax Credit to homeowners. Michigan renters qualify through the rent-based property-tax equivalent, which is 23% of the listed $10,560 annual rent and yields a $760.79 credit."
+us,scenario_045,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model derived approximately $760.78 from the correct renter-credit formula and correctly found no Michigan EITC, but then submitted $884. It added an unsupported $123.22 after its own computation established that the Homestead Property Tax Credit was the only refundable credit."
+us,scenario_045,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model focused on the Michigan EITC and Home Heating Credit while omitting the rent-based Homestead Property Tax Credit calculation. Its $1,209 submission has no supporting computation; the listed rent and $36,275.59 of household resources produce $760.79 instead."
+us,scenario_045,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model treated the zero Michigan EITC as exhausting refundable state credits and omitted the Michigan Homestead Property Tax Credit. The renter pathway produces $2,428.80 of countable property tax and a $760.79 refundable credit."
+us,scenario_045,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model interpreted the absence of directly listed property tax as disqualifying, ignoring that Michigan converts 23% of a renter's listed rent into countable property tax. The $10,560 rent therefore supplies $2,428.80 for the Homestead Property Tax Credit calculation."
+us,scenario_045,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model correctly concluded that federal and Michigan EITC equal zero, then invented a $1,089 Working Families Tax Credit amount despite acknowledging that result. It omitted the separate Homestead Property Tax Credit, whose renter formula yields $760.79."
+us,scenario_045,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model improperly reduced household resources from $36,275.59 to $15,068 by subtracting the listed employer-sponsored insurance premiums and also used 20% rather than 23% of rent. It further invented a positive childless federal EITC at $36,276, producing both a false Michigan EITC and an overstated homestead credit."
+us,scenario_045,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly used 23% of rent but improperly reduced household resources to $15,068 by subtracting the $21,208 employer-sponsored insurance premiums. Michigan household resources remain $36,275.59 for this calculation, producing a $760.79 credit rather than $1,167.97."
+us,scenario_045,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits Michigan's refundable Homestead Property Tax Credit for renters. The listed $10,560 rent is sufficient to establish $2,428.80 of countable property tax, and the household-resource test yields $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model treated the zero federal EITC, and therefore zero Michigan EITC, as determining all state refundable credits. It failed to calculate the independent Michigan Homestead Property Tax Credit available to this renter, which equals $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model used 20% instead of 23% for the renter property-tax equivalent and also included a Michigan EITC based on an inapplicable positive federal EITC. At $36,276 with no qualifying children, the EITC is zero, while the correctly calculated Homestead Property Tax Credit is $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model assigned a positive federal childless EITC at $36,276 and took 30% of it for Michigan, even though the childless EITC has fully phased out at this income. It also omitted the separate rent-based Homestead Property Tax Credit of $760.79."
+us,scenario_045,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model recognized the renter's Homestead Property Tax Credit but miscomputed its amount. Applying the 23% rent conversion, the household-resource exemption, and the 60% factor yields $760.79 rather than $695.48."
+us,scenario_045,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used 20% rather than 23% of rent and calculated $570.70, but then contradicted even that calculation by submitting zero. The correct renter property-tax equivalent is $2,428.80 and the resulting refundable credit is $760.79."
+us,scenario_045,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to identify the Michigan Homestead Property Tax Credit from the explicit rent and household-resource facts. Renters qualify through the 23%-of-rent property-tax equivalent, producing $760.79 here."
+us,scenario_045,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required a dependent or another unspecified condition for any Michigan refundable credit. A childless renter can qualify for the Homestead Property Tax Credit, and this household's rent and resources produce $760.79."
+us,scenario_045,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model used 20% of rent as the Michigan renter property-tax equivalent instead of 23%. Using $2,428.80 rather than $2,112 as countable property tax raises the correctly calculated credit to $760.79."
+us,scenario_045,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model improperly reduced household resources to $15,068 by subtracting the $21,208 employer-sponsored insurance premiums and invented a positive Michigan EITC. Household resources are $36,275.59 for this credit, the childless EITC is zero at this income, and the homestead credit is $760.79."
+us,scenario_045,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly treated being childless as disqualifying for every Michigan refundable credit. The Homestead Property Tax Credit does not require a qualifying child, and the renter calculation yields $760.79."
+us,scenario_045,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted the renter eligibility pathway for Michigan's Homestead Property Tax Credit. The listed rent converts to $2,428.80 of countable property tax and generates a $760.79 refundable credit."
+us,scenario_045,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model improperly used $15,068 of household resources, a 3.5% resource percentage, and a positive federal and Michigan EITC. The applicable calculation uses $36,275.59 of resources and a 3.2% exemption, while the EITC is zero, producing total refundable credits of $760.79."
+us,scenario_045,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,"The model incorrectly deducted the $5,200 child-support expense from Michigan household resources. The credit calculation uses $36,275.59 of household resources rather than $31,076, yielding $760.79 instead of $861."
+us,scenario_045,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The zero answer omits the Michigan Homestead Property Tax Credit available to renters. Applying the rent-based property-tax equivalent and household-resource test gives $760.79.
+us,scenario_045,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model incorrectly treated the Homestead Property Tax Credit as only an amount offsetting state tax liability and concluded that no refundable portion remained. The Michigan credit itself is refundable, and the renter calculation produces $760.79 independently of the zero EITC."
+us,scenario_045,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for state_refundable_credits. It therefore failed the required output contract before any substantive tax calculation could be evaluated.
+us,scenario_045,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model used both the wrong renter property-tax percentage, 20% instead of 23%, and the wrong household-resource percentage, 3.5% instead of 3.2%. The applicable parameters produce a $760.79 Homestead Property Tax Credit."
+us,scenario_045,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model disregarded the explicitly listed pre-subsidy annual rent and treated the renter as having no qualifying rent paid. That $10,560 rent is the housing input used to derive $2,428.80 of countable property tax and a $760.79 credit."
+us,scenario_045,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model used 17% instead of 23% of rent for the renter property-tax equivalent and 3.5% instead of 3.2% for the household-resource exemption. It also cited an outdated Michigan EITC percentage, though the federal EITC is zero; the correct homestead result is $760.79."
+us,scenario_045,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model asserted that the listed facts did not qualify for a Homestead Property Tax Credit, overlooking the explicit Michigan residence, rent, and household resources. Those facts establish renter eligibility and produce a refundable credit of $760.79."
+us,scenario_045,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model omitted Michigan's refundable Homestead Property Tax Credit without applying its renter pathway. The listed rent and household resources yield $760.79.
 us,scenario_046,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model asserted that the child was “likely eligible” without testing Oklahoma's income limits or any categorical pathway. At 2.79 times FPL, child1 fails the regular children's Medicaid limit, CHIP limit, and all other applicable categories, yielding medicaid_category = NONE and value = 0."
 us,scenario_046,child2_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model assumed that being a child made Child 2 eligible without testing Oklahoma's categorical and income requirements. At 2.79 times FPL, Child 2 exceeds all applicable child Medicaid limits and qualifies through no other pathway, so the correct value is 0."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own calculation reached approximately $488 after the overtime deduction and $4,400 CTC, but it submitted $3,465 without a computation supporting that substitution. Using the exact deduction and brackets yields $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $14,838.71 qualified-overtime deduction and used a $28,700 standard deduction instead of $32,200. It also invented a $2,500 AOTC despite zero listed qualified education expenses and understated the two-child CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction and used the wrong standard deduction and $2,000-per-child CTC instead of $2,200. Its submitted $4,669 also contradicts its own stated result of about $2,777."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $14,838.71 qualified-overtime deduction, then submitted approximately tentative tax without applying the child credits it discussed. The correct taxable income is $44,964.29 and the full $4,400 CTC reduces $4,899.71 to $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model essentially found the correct taxable-income path but reported tentative tax before the Child Tax Credit. This output requires subtracting the $4,400 nonrefundable CTC, leaving $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction, used an understated standard deduction, and applied a $4,000 CTC instead of the 2026 amount of $4,400. Those errors inflated taxable income and the post-credit liability."
-us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $12,200 has no connection to the model’s own tentative-tax or credit calculations and improperly invokes employer insurance and itemized adjustments after selecting the standard deduction. The required sequence produces $4,899.71 before credits and $499.71 after the $4,400 CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used obsolete personal exemptions and pre-TCJA brackets instead of the applicable 2026 rules. It also omitted the $32,200 standard deduction, mishandled the overtime deduction, and allowed only $2,000 of CTC rather than $4,400."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reconstructed wages from hourly details even though $92,000 was the controlling annual wage total, treated employer coverage premiums as a wage adjustment, and used personal exemptions and itemized deductions instead of the standard-plus-overtime deductions. It also allowed only $2,000 of CTC for two children instead of $4,400."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated calculation omits the $14,838.71 qualified-overtime deduction and uses a $31,200 standard deduction rather than $32,200. Its $8,518 result is also incompatible with subtracting the stated $4,000 child credit from tax on the stated taxable income."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $9,208 employer-sponsored insurance premium from already stated gross wages and applied personal exemptions and itemized deductions. It also used only $2,000 of total CTC rather than $4,400 and omitted the qualified-overtime deduction."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the employer-sponsored insurance premium, applied personal exemptions and itemized deductions, and omitted the qualified-overtime deduction. It also invented a $1,500 nonrefundable AOTC despite zero qualified education expenses and allowed only $2,000 of CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The unexplained $1,858.40 does not reflect the traced $44,964.29 taxable income and $4,400 CTC. Applying the standard and overtime deductions gives $4,899.71 of tentative tax and $499.71 after nonrefundable credits."
-us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model explicitly relied on itemized deductions and personal exemptions, neither of which is the operative deduction path. The household instead uses the $32,200 standard deduction plus the $14,838.71 overtime deduction, followed by a $4,400 CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response could not be evaluated as a substantive tax calculation."
-us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invented an American Opportunity Credit even though unlisted qualified education expenses are zero and then forced the liability to zero. Only the $4,400 CTC applies against $4,899.71 of tentative tax, leaving $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $6,794 answer is consistent with taxing income after a standard deduction while omitting the $14,838.71 qualified-overtime deduction and the $4,400 CTC. No AOTC applies because qualified education expenses are zero."
-us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model capped the qualified-overtime deduction at $12,500 instead of deducting the traced $14,838.71. That left taxable income $2,338.71 too high; the exact taxable income is $44,964.29 and post-CTC tax is $499.71."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction and did not apply the $4,400 CTC to the applicable tentative tax. Its reference to an AOTC phaseout is irrelevant because qualified education expenses are zero, so no AOTC is generated."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used personal exemptions, obsolete 10%/15% brackets, and an itemized-deduction path instead of the $32,200 standard deduction plus $14,838.71 overtime deduction. It also allowed only $2,000 of CTC for two children rather than $4,400."
-us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a pre-TCJA regime and understated the two-child CTC as $2,000. Under the applicable 2026 rules, taxable income is $44,964.29, tentative tax is $4,899.71, and the CTC is $4,400."
-us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly itemized medical premiums and other expenses to reduce taxable income to about $34,706 instead of using the $32,200 standard deduction alongside the overtime deduction. The correct tentative tax exceeds the $4,400 CTC by $499.71, so the credit does not reduce liability to zero."
-us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, constituting a missing-output failure rather than a substantive computation."
-us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model provided no supporting computation, and $9,999 does not reflect the required deduction-and-credit sequence. The traced calculation yields $44,964.29 of taxable income, $4,899.71 of tentative tax, and $499.71 after the CTC."
-us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the qualified-overtime deduction, improperly itemized health premiums and over-the-counter expenses, and invented an AOTC despite zero qualified education expenses. It also reversed the AOTC refundable split; regardless, no AOTC is available here."
-us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model omitted the qualified-overtime deduction and invented a $2,500 AOTC despite zero qualified education expenses. Its submitted $8,838 also contradicts its own stated post-credit result of $342 and therefore is not the result of its described calculation."
-us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the refundable CTC earnings formula and per-child cap as an automatic refund even though the household’s tax liability absorbs the available CTC nonrefundably. Its stated calculation produces a $3,400 cap but it submitted $3,200, adding an internal arithmetic inconsistency."
-us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model invented sufficient qualified education expenses to maximize the AOTC despite the instruction that every unlisted numeric input is zero. Student eligibility facts without qualified expenses produce no AOTC and therefore no $1,000 refundable portion."
-us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model treated the $4,400 Child Tax Credit and a maximum AOTC as refundable amounts rather than separating credits used against liability from refundable portions. Qualified education expenses are zero, and the available CTC is absorbed nonrefundably, so neither produces refundable credits."
-us,scenario_046,federal_refundable_credits,claude-sonnet-4.6,llm_error,other,False,"The model correctly concluded that the tax liability absorbs the Child Tax Credit, but then awarded the maximum refundable AOTC based only on enrollment and documentation status. Because no qualified education expenses are listed, the prompt sets the AOTC expense base to zero."
-us,scenario_046,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model awarded a $1,000 refundable AOTC solely because the head satisfies student-status requirements. The missing qualified education expense input is zero under the prompt, so the AOTC and its refundable portion are both zero."
-us,scenario_046,federal_refundable_credits,gemini-3.5-flash,llm_error,other,False,"The model assumed that AOTC eligibility automatically yields the $2,500 maximum credit. With no qualified tuition or related expenses listed, the credit base is zero and the 40% refundable portion is also zero."
-us,scenario_046,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, so it failed the required structured-output contract."
-us,scenario_046,federal_refundable_credits,inkling,llm_error,other,False,"The model understated the federal tax liability available to absorb the Child Tax Credit and therefore created a nonexistent $725 residual refundable CTC. The liability absorbs the available CTC nonrefundably, leaving refundable CTC of zero."
-us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, so it failed the required structured-output contract."
-us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model invented qualified education expenses and awarded a $1,000 refundable AOTC even though unlisted expenses are zero. It then submitted $1,500 despite its own component calculation totaling $1,000, introducing an additional unexplained $500."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its own stated derivation produced approximately $488 after the overtime deduction and two $2,200 CTCs, but it submitted $3,465 without any computation supporting that adjustment. Using the exact deduction, brackets, and credits yields $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $14,838.71 qualified-overtime deduction and used a $28,700 standard deduction instead of $32,200. It also awarded an AOTC despite qualified education expenses being unlisted and therefore zero, and used outdated CTC amounts instead of $2,200 per child."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the qualified-overtime deduction and used an understated standard deduction, then applied outdated $2,000-per-child CTC amounts. Its submitted $4,669 also contradicts its own stated result of approximately $2,777."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It failed to subtract the $14,838.71 qualified-overtime deduction and treated the pre-credit regular tax as the requested output instead of subtracting the two $2,200 nonrefundable CTCs. It also discussed an AOTC even though unlisted qualified education expenses are zero."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"It stopped at an approximate tentative tax and explicitly reported the pre-child-credit amount. The requested output subtracts $4,400 of nonrefundable CTC from exact tentative tax of $4,899.71, leaving $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It omitted the qualified-overtime deduction, used an understated standard deduction, and used $2,000 rather than $2,200 for each child's 2026 CTC. Those errors inflated both taxable income and the liability remaining after credits."
+us,scenario_046,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It omitted the qualified-overtime deduction and invented an AOTC despite zero listed education expenses, then submitted $12,200 even though its own tentative-tax calculation was only $6,696.36. The submitted figure follows no valid tax or credit computation."
+us,scenario_046,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied a pre-TCJA personal-exemption regime and 10%/15% brackets instead of the applicable 2026 rules, omitted the qualified-overtime and standard deductions, and allowed only $2,000 of CTC. The applicable computation uses no personal exemptions and provides $4,400 of CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reconstructed wages from hourly details even though $92,000 was the controlling annual wage total, improperly subtracted employer-sponsored premiums from wages, and used personal exemptions and itemized deductions instead of the $32,200 standard plus qualified-overtime deductions. It also allowed only $2,000 total CTC rather than $4,400."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"Its stated inputs imply regular tax near $6,800 before its claimed $4,000 CTC, so the submitted $8,518 exceeds even its own pre-credit tax. It also omitted the $14,838.71 overtime deduction and understated the standard deduction and CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It improperly deducted the $9,208 employer-sponsored insurance premium from wages and applied personal exemptions and estimated itemized deductions instead of the standard and qualified-overtime deductions. It also reduced tax by only $2,000 for two children instead of $4,400."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It improperly excluded the employer-sponsored insurance premium from wages, applied personal exemptions and itemized deductions, and awarded a $1,500 nonrefundable AOTC despite zero qualified education expenses. It also used only $2,000 total CTC rather than two $2,200 credits."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer reflects a taxable-income or credit computation that did not combine the $32,200 standard deduction, $14,838.71 overtime deduction, and $4,400 CTC. Those exact steps reduce $4,899.71 of tentative tax to $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its reference to itemized deductions, personal exemptions, and child credits applies the wrong 2026 tax regime. The household instead uses the standard deduction plus the qualified-overtime deduction, has no personal exemptions, and receives $4,400 of CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for the requested variable.
+us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"It invented an American Opportunity Credit despite zero listed qualified education expenses and defaulted the residual liability to zero. The two $2,200 CTCs reduce $4,899.71 of tentative tax only to $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Its $6,794 answer corresponds to regular tax calculated without the $14,838.71 overtime deduction and without subtracting the $4,400 CTC. It also invoked an AOTC even though qualified education expenses are zero."
+us,scenario_046,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It capped the qualified-overtime deduction at $12,500 instead of deducting the trace amount of $14,838.71. That left taxable income $2,338.71 too high and produced $280.65 too much tax after the $4,400 CTC."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"It asserted an AOTC phaseout despite AGI being below the joint phaseout range and, more fundamentally, zero listed qualified education expenses. Its $8,500 also fails to apply the $14,838.71 overtime deduction and $4,400 CTC that produce $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It applied a post-sunset personal-exemption and 10%/15% bracket regime, omitted the qualified-overtime deduction, and used only $2,000 total CTC. The applicable 2026 computation uses the $32,200 standard deduction, 10%/12% brackets at this income, and $2,200 per child."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It incorrectly applied personal exemptions, itemized deductions, post-sunset brackets, and $1,000-per-child CTCs. It omitted the qualified-overtime deduction and the applicable $2,200 credit for each child."
+us,scenario_046,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"It used a pre-TCJA regime and only $2,000 total CTC, producing an unsupported $23,508 taxable-income figure. The applicable deductions yield taxable income of $44,964.29, and two $2,200 credits leave $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It incorrectly placed the overtime deduction in AGI and used inflated itemized medical deductions instead of the larger $32,200 standard deduction. That understated tentative tax enough for the $4,400 CTC to erase it, whereas exact tentative tax is $4,899.71 and leaves $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for the requested variable.
+us,scenario_046,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The unexplained $9,999 exceeds the tax generated by the household's income before credits and does not reflect the qualified-overtime deduction or $4,400 CTC. The specified computation yields $499.71."
+us,scenario_046,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It double counted the same $12,000 health premiums under separate input labels and treated over-the-counter expenses as deductible medical costs, causing itemized deductions to exceed the standard deduction incorrectly. Using the $32,200 standard deduction produces $4,899.71 of tentative tax, which the $4,400 CTC does not fully eliminate."
+us,scenario_046,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It omitted the qualified-overtime deduction, double counted the same $12,000 premium, included nondeductible over-the-counter expenses, and awarded an AOTC despite zero qualified education expenses. It also reversed the AOTC split: the maximum credit is 40% refundable and 60% nonrefundable, not $1,500 refundable and $1,000 nonrefundable."
+us,scenario_046,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"It omitted the qualified-overtime deduction and awarded a $2,500 AOTC despite zero qualified education expenses. Its submitted $8,838 also directly contradicts its own stated after-credit result of $342 and exceeds its stated $6,842 regular tax."
+us,scenario_046,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the refundable CTC cap and earned-income formula as creating a refund even though the household's tax liability absorbs the available CTC. Its submitted $3,200 also contradicts its own $3,400 cap calculation."
+us,scenario_046,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented sufficient qualified education expenses to maximize the American Opportunity Credit. Because tuition and other qualified expenses are unlisted, they are $0 under the prompt, so the refundable AOTC is $0."
+us,scenario_046,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model improperly counted the Child Tax Credit and a maximized AOTC as refundable credits. Tax liability absorbs the CTC, and the prompt supplies $0 of qualified education expenses, so neither produces a refundable amount."
+us,scenario_046,federal_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly found no refundable CTC or EITC but awarded the maximum $1,000 refundable AOTC solely from the head's student eligibility facts. AOTC also requires qualified education expenses, which are $0 because none are listed."
+us,scenario_046,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model awarded a $1,000 refundable AOTC from student status without applying the qualified-expense requirement. The prompt makes unlisted tuition and other qualified education expenses $0, yielding no AOTC."
+us,scenario_046,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model assumed the head receives the maximum AOTC merely because the eligibility conditions are satisfied. With no listed qualified education expenses, the credit base is $0 and the refundable portion is $0."
+us,scenario_046,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_refundable_credits, violating the required output contract."
+us,scenario_046,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model understated pre-credit federal tax liability and therefore manufactured $725 of unused CTC. The household's liability absorbs the available CTC, leaving refundable CTC of $0."
+us,scenario_046,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for federal_refundable_credits, violating the required output contract."
+us,scenario_046,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model understated the federal tax liability to $3,871.01 and treated the resulting $528.99 of CTC as refundable. The available CTC is fully absorbed by liability, so no refundable CTC remains."
+us,scenario_046,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own component calculation totals $1,000—$1,000 of asserted AOTC and $0 from CTC and EITC—but it submitted $1,500. It also wrongly awarded the AOTC because no qualified education expenses are listed, making the correct refundable AOTC $0."
 us,scenario_046,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_046,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly computed $5,704 of Social Security tax and $1,334 of Medicare tax but then added them incorrectly as $10,038 and invented a $518.40 Oklahoma payroll-tax adjustment. Oklahoma contributes no mandatory employee payroll tax here, so the two federal components total $7,038."
 us,scenario_046,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the $9,208 employer-sponsored insurance premium from FICA wages. The facts do not identify that premium as a pre-tax employee payroll deduction, so Social Security and Medicare apply to the full $92,000 of wages and total $7,038."
@@ -3014,36 +3169,38 @@ us,scenario_046,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The mod
 us,scenario_046,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's reasoning calculated the exact $7,038 total but submitted $8,438 as the output value. It failed to carry its computed result into the required numeric field, creating a $1,400 answer inconsistency."
 us,scenario_046,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_046,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model selected the $12,700 Oklahoma standard deduction instead of the engine-selected $22,429.78 itemized deduction. It also used a $200 credit instead of 5% of the $4,400 federal CTC, or $220, and its submitted $3,364 contradicted its own $3,022 calculation."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an inapplicable $14,600 deduction and a 5.85% Oklahoma rate, then inserted an unsupported residual-credit estimate to reach $1,614. The required computation uses the Oklahoma joint-filer brackets and subtracts the specific $220 nonrefundable state credit."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction instead of the $22,429.78 itemized deduction. It then approximated the Oklahoma child credit rather than calculating the required $220 from 5% of the $4,400 federal CTC."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model replaced the engine-selected $22,429.78 itemized deduction with an asserted $17,000 cap and did not apply the $220 nonrefundable Oklahoma child credit. Its bracket total also failed to follow the applicable joint-filer schedule."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the standard deduction instead of the selected itemized deduction and then compressed the bracket-and-credit calculation into an unsupported $2,500 estimate. The correct sequence produces $2,959.14 before the $220 credit and $2,739.14 after it."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated Oklahoma as conforming to an approximately $30,000 federal joint standard deduction and rejected itemizing on that basis. Oklahoma's computation selects $22,429.78 of itemized deductions and also applies the $220 nonrefundable child care/child tax credit, which the model omitted."
-us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"After falsely stating that Oklahoma has no income tax, the model used an approximate $20,879 itemized deduction rather than $22,429.78 and never completed the exact bracket calculation. It also failed to subtract the $220 nonrefundable Oklahoma child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $51,795.62 using an unspecified federal-itemized-deduction calculation, rather than following the Oklahoma deduction inputs in the engine trace. That excessive reduction drove its bracket tax down to $2,105.29 and bypassed the required $220 credit step."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model correctly identified $65,573 as its deduction-based taxable-income figure but subtracted only $100 of Oklahoma child credit. The state credit is 5% of the household's $4,400 federal CTC, so the required nonrefundable reduction is $220."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no computation supporting $3,154. The engine's specified deduction, joint-bracket, and $220 nonrefundable-credit sequence yields $2,739.14, so the answer omitted or miscomputed at least one of those required steps."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model asserted an estimated $2,105 tax without identifying the itemized-deduction amount, bracket calculation, or nonrefundable credit. It failed to reproduce the engine-selected $22,429.78 deduction and the final $220 credit reduction."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly lowered Oklahoma AGI to $82,795 and deducted $26,999, producing taxable income of $51,796. It also used a $100 child credit instead of the required $220 calculated from 5% of the $4,400 federal CTC."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model gave only a conclusory figure and did not reproduce the traced deduction and credit calculation. Applying the engine-selected $22,429.78 itemized deduction, Oklahoma joint brackets, and $220 credit yields $2,739.14 rather than $2,276.01."
-us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model expressly used the standard deduction, while the engine selects the larger $22,429.78 itemized deduction. Its unexplained nonrefundable-credit calculation also failed to arrive at the traced $220 reduction."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model chose the $12,700 Oklahoma standard deduction after understating the available itemized deductions, instead of using the larger $22,429.78 itemized amount. It also submitted $3,364 despite its own final computation of $3,022 and used a $200 credit instead of $220."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an inapplicable $14,600 deduction and a nonexistent 5.85% Oklahoma rate, then inserted an unsupported credit adjustment to reach $1,614. Oklahoma requires the joint-filer bracket calculation followed by the $220 nonrefundable state credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction instead of the larger $22,429.78 itemized deduction. It then estimated the state credit rather than calculating the $220 credit from the $4,400 federal child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model reduced the household's itemized deductions to a $17,000 cap and failed to use the traced $22,429.78 deduction. It also stopped at an incorrectly computed bracket tax without subtracting the $220 nonrefundable child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used the standard deduction rather than the larger itemized deduction and compressed the bracket and credit calculations into an unsupported approximation. The exact regular tax is $2,959.14 and the exact nonrefundable credit is $220, yielding $2,739.14."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated Oklahoma as conforming to an approximately $30,000 federal joint standard deduction and therefore rejected the applicable $22,429.78 itemized deduction. It also omitted the $220 Oklahoma child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"Although the model recognized that itemizing exceeded Oklahoma's standard deduction, it only approximated the taxable-income and bracket calculation and did not apply the exact $220 nonrefundable credit. Its contradictory statement that Oklahoma has no income tax further shows that it did not execute a coherent Oklahoma calculation."
+us,scenario_046,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $51,795.62 through unspecified federal itemized deductions, rather than using the traced $22,429.78 Oklahoma itemized deduction and four exemptions. It also omitted the $220 nonrefundable state credit from a correctly derived regular-tax amount."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model constructed a $17,000 cap plus a $5,430 medical deduction instead of using the traced $22,429.78 itemized deduction. It then used a $100 credit based on one $2,000 federal child credit rather than the $220 state credit derived from the household's $4,400 federal child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unsupported $3,154 answer does not reflect the required $2,959.14 regular tax followed by the $220 nonrefundable credit. The model supplied no deduction, bracket, or credit computation supporting its number."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $2,105 estimate implies excessive deductions relative to the traced $22,429.78 itemized amount. It also failed to identify and subtract the exact $220 Oklahoma nonrefundable child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly lowered Oklahoma AGI to $82,795 and deducted $26,999 rather than starting from $92,003 and using the traced $22,429.78 itemized deduction. It also subtracted only $100 instead of the $220 credit derived from the $4,400 federal child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model provided no figures for its itemized deductions, exemptions, bracket tax, or nonrefundable credit. Its $2,276.01 answer does not implement the traced $2,959.14 regular tax less the $220 credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model expressly used the standard deduction instead of the larger $22,429.78 itemized deduction. It also failed to show the exact $2,959.14 bracket tax and $220 nonrefundable credit needed for the output."
 us,scenario_046,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model inserted a zero placeholder instead of performing the fully specified Oklahoma calculation. The provided facts support the traced deduction, bracket tax, and $220 credit computation yielding $2,739.14."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly treated the federal American Opportunity Credit as a nonrefundable Oklahoma tax credit. It also failed to show the engine-selected $22,429.78 itemized deduction and the actual $220 Oklahoma child care/child tax credit."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction rather than the engine-selected $22,429.78 itemized deduction. Although it correctly applied the $220 credit, its preceding regular-tax amount of $3,064 did not match the traced bracket tax of $2,959.14."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model introduced an overtime deduction into Oklahoma taxable income without carrying through the traced Oklahoma itemized-deduction computation. Its $4,223.29 figure also omitted the explicit $220 nonrefundable child credit reduction."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model imposed a $17,000 itemized-deduction cap instead of using the engine-selected $22,429.78 amount. Although it subtracted the correct $220 credit, the wrong deduction base produced an overstated $3,018 regular tax."
-us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied a flat 4.5% rate instead of Oklahoma's joint-filer bracket computation. It also used two $100 child credits rather than the single traced Oklahoma credit of $220 based on 5% of the $4,400 federal CTC."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model provided no deduction, bracket, or credit calculation supporting $3,800. It failed to reproduce the traced $2,959.14 regular tax and the subsequent $220 nonrefundable credit."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used $26,309 of federal-style itemized deductions rather than the engine-selected Oklahoma amount of $22,429.78. It also stopped at bracket tax and omitted the $220 nonrefundable Oklahoma child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model invented $45,216 of taxable income and applied a flat 4.5% rate. Oklahoma instead uses the traced deduction and exemption calculation, graduated joint-filer brackets, and a final $220 nonrefundable child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced Oklahoma AGI to about $77,164 and then applied another approximate $15,000 standard deduction. It also stated that no state credits applied, omitting the $220 nonrefundable Oklahoma child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model substituted a zero placeholder instead of performing the Oklahoma calculation from the supplied facts. The required computation produces $2,959.14 before credits and $2,739.14 after the $220 nonrefundable credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated the federal American Opportunity Credit as a nonrefundable Oklahoma tax credit, which is not the state credit used here. It also failed to calculate the $220 Oklahoma child care/child tax credit from the $4,400 federal child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used Oklahoma's $12,700 standard deduction instead of the larger $22,429.78 itemized deduction. Although it correctly used the $220 credit, its overstated $3,064 regular tax led to the wrong result."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model invoked an overtime deduction without reporting the resulting taxable income and produced tax far above the traced bracket liability. It failed to use the $22,429.78 itemized deduction and the final $220 nonrefundable credit calculation."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model capped itemized deductions at $17,000 instead of using the traced $22,429.78 amount. Its $220 credit was correct, but it applied that credit to an overstated regular-tax estimate rather than $2,959.14."
+us,scenario_046,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied a flat 4.5% approximation instead of Oklahoma's joint-filer graduated brackets. It also used two $100 credits rather than the single $220 Oklahoma credit derived from the $4,400 federal child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model gave an unsupported $3,800 figure and did not identify any deduction, exemption, bracket, or credit calculation. It omitted the required sequence of $2,959.14 regular tax less the $220 nonrefundable credit."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model overstated itemized deductions at $26,309 by including a medical deduction not present in the traced Oklahoma deduction amount. It also stopped at regular tax and omitted the $220 nonrefundable child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used $26,309 of itemized deductions instead of the traced $22,429.78 and therefore understated taxable income. It also omitted the $220 nonrefundable Oklahoma credit after computing bracket tax."
+us,scenario_046,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model invented $45,216 of taxable income and applied a flat 4.5% rate rather than Oklahoma's joint-filer graduated brackets. It did not apply the $220 nonrefundable child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced Oklahoma AGI to about $77,164 and then applied another approximate standard deduction, instead of starting with $92,003 and using $22,429.78 of itemized deductions. It also expressly omitted the applicable $220 state credit."
 us,scenario_046,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
-us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the $12,700 standard deduction, omitted all four $1,000 Oklahoma exemptions from its stated taxable-income calculation, and rejected the applicable state credit. The computation requires the engine-selected itemized deduction, $4,000 of exemptions, and the $220 nonrefundable child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model supplied only the label and an unsupported $3,773 value. It did not perform the traced deduction, graduated-bracket, or $220 nonrefundable-credit calculation."
-us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as $94,004 despite wages and interest totaling $92,003, used the $12,700 standard deduction, and omitted the four $1,000 exemptions. It also failed to subtract the $220 nonrefundable Oklahoma child credit."
-us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model used an obsolete or inapplicable 0.5%-to-5% Oklahoma rate schedule rather than the applicable 2026 joint-filer brackets. It also asserted that no nonrefundable credit applied, omitting the traced $220 Oklahoma child care/child tax credit."
+us,scenario_046,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted all four $1,000 Oklahoma exemptions and used the $12,700 standard deduction instead of the larger $22,429.78 itemized deduction. It also incorrectly concluded that no nonrefundable state credit applied, omitting $220."
+us,scenario_046,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model supplied only an unsupported $3,773 amount. It did not perform the traced itemized-deduction, exemption, bracket, and $220 credit calculation."
+us,scenario_046,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model chose the $12,700 standard deduction after incorrectly deciding itemizing was not beneficial, rather than using $22,429.78 of itemized deductions. It also set the Oklahoma child care/child tax credit to zero instead of subtracting $220."
+us,scenario_046,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model overstated federal AGI as roughly $94,004 instead of $92,003, omitted the four $1,000 exemptions, and used the standard deduction rather than the larger itemized deduction. It also failed to subtract the $220 nonrefundable Oklahoma credit."
+us,scenario_046,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,The model used an obsolete or inapplicable 0.5%-to-5% Oklahoma bracket schedule instead of the 2026 joint-filer rates used in the trace. It also incorrectly concluded that no nonrefundable credit applied and omitted the $220 child care/child tax credit.
 us,scenario_046,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_046,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_048,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated age and assets below $2,000 as sufficient for Ohio aged Medicaid while never establishing that the head passed the applicable income test. The head's wages and Social Security total $34,626, SSI is zero, and no Medicaid category applies."
@@ -3052,42 +3209,45 @@ us,scenario_048,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eli
 us,scenario_048,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly claimed that most Social Security survivor benefits are excluded under SSI-related Medicaid counting and conflated Medicare Savings Program eligibility with regular Medicaid eligibility. Age, low assets, and possible QMB, SLMB, or QI status do not establish eligibility for the requested Medicaid output, and no Medicaid category applies."
 us,scenario_048,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model labeled the head's income ""very low"" and inferred Medicaid eligibility directly from age and residence. The head's MAGI income is 2.17 times FPL, SSI is zero, and age 85 alone does not supply a qualifying Medicaid pathway."
 us,scenario_048,head_medicaid_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted eligibility based on age and income without identifying or applying any Ohio Medicaid pathway. The head's age does not independently confer eligibility, and the income facts result in no qualifying Medicaid category."
+us,scenario_048,head_medicare_eligible,ox-alpha,llm_error,other,False,"The model correctly identified the age-based Medicare eligibility pathway and explicitly concluded that the head is eligible, but then encoded that conclusion as value = 0. Under the required binary contract, Medicare eligible is 1, so the numeric submission contradicted its own reasoning."
 us,scenario_048,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model identified the correct payroll-tax base and components but miscalculated their amounts, rounded the annual liability to a whole dollar, and then submitted $782 despite its own stated total of $586. Applying 6.2% and 1.45% to $7,656 yields $474.67 and $111.01, respectively, for $585.68."
 us,scenario_048,snap,claude-opus-4.7,llm_error,thresholds_rates,False,The model correctly calculated that the benefit formula produced $0 but then applied the $23 minimum allotment to an income-ineligible household. It also submitted $23 as an annual amount after stating that $23 was monthly and that twelve months would equal $276.
 us,scenario_048,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model explicitly found that net income exceeded its stated eligibility limit and that the household was ineligible, yet submitted $2,376. It applied a positive allotment after its own eligibility calculation had already required a $0 benefit."
 us,scenario_048,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model treated age 85 as bypassing the controlling income screen and assumed the minimum allotment remained available. The household’s $34,626 income exceeds the $17,420 eligibility threshold, so no minimum benefit is payable."
 us,scenario_048,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted that medical and shelter deductions reduced net income to roughly $0–$100 without performing the arithmetic. The listed medical expenses yield only a small excess-medical deduction, and $650 monthly rent is below half of the roughly $2,519 monthly income after the deductions it identified, so there is no excess-shelter deduction and no near-maximum allotment."
 us,scenario_048,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled $34,626 of annual wages and Social Security as “very low countable income” and assigned an estimated $225 monthly benefit without applying the SNAP income threshold. That income exceeds the $17,420 limit for a one-person household, producing $0."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"Its stated calculation produces $30,830, but it submitted $33,223 with no computation supporting that number. It also overstated AGI by $721 relative to the traced $246,663.72, failing to reflect the deductible traditional IRA contribution used in the reference computation."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified-overtime deduction and used a $29,200 standard deduction instead of $32,200. It then applied the ordinary rate schedule incorrectly to its own $217,364 taxable-income figure, producing $48,837 rather than separating the $6,208 of qualified dividends and taxing them preferentially."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified-overtime deduction and used an estimated $31,500 standard deduction instead of $32,200. It also included $27 of Additional Medicare Tax even though the requested federal-income-tax output excludes payroll taxes."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"It omitted the $25,000 qualified-overtime deduction and used only a $30,300 standard deduction, leaving taxable income far too high. Its submitted $53,503 also contradicts its own stated tax calculation of about $37,055."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It correctly identified the higher 2026 SALT cap but then added deductions to about $46,081 even though the listed property tax, mortgage interest, and gifts total only $26,736 before limitations. More importantly, $47,500 is not the tax produced by the married-joint rate schedule on its asserted taxable income of about $201,300."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It never applied the $25,000 qualified-overtime deduction and used a $30,000 standard deduction instead of $32,200. Those errors left taxable income at $217,385 rather than $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"It omitted the $25,000 qualified-overtime deduction and then abandoned its own bracket calculation of roughly $38,831 to submit $62,700. Additional Medicare Tax and unspecified phase-outs cannot be added to this federal-income-tax output and do not reconcile that unsupported increase."
-us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It wrongly subtracted the separately listed $16,408 employer-sponsored insurance premium from gross wages even though the traced employment income is $237,564 after only the traditional 401(k) deferral. It also applied a fictional TCJA sunset with personal exemptions and pre-TCJA brackets, and omitted the $25,000 overtime deduction and $32,200 standard deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It double-counted the $16,408 ESI premium as an additional wage exclusion, used itemized deductions and personal exemptions under an assumed TCJA sunset, and omitted the $25,000 overtime deduction. The applicable computation instead uses $246,663.72 of AGI and $57,779.77 of deductions."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its answer implies taxable income of about $229,000, whereas the traced standard and overtime deductions reduce taxable income to $188,883.95. It supplied no calculation identifying either the $32,200 standard deduction or the $25,000 overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It wrongly excluded the $16,408 ESI premium again from wages, then assumed pre-TCJA itemization and personal exemptions. It also omitted the $25,000 qualified-overtime deduction that drives taxable income down to $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It used post-sunset itemized deductions and personal exemptions to reach $194,141 of taxable income. The 2026 computation instead uses the $32,200 standard deduction plus the $25,000 overtime deduction and yields taxable income of $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It wrongly subtracted the $16,408 ESI premium from gross wages, used a $30,000 standard deduction, and omitted the $25,000 qualified-overtime deduction. These offsetting errors produced $200,977 of taxable income instead of $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its terse explanation never applies the $25,000 qualified-overtime deduction or establishes the traced taxable income of $188,883.95. The submitted amount is consistent with taxing a materially larger base than the base remaining after the $32,200 standard deduction and overtime deduction."
-us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,It returned no numeric value or explanation for the requested output.
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its $20,460 answer is far below the $29,612.71 ordinary tax plus $931.20 qualified-dividend tax produced by the married-joint schedule on $188,883.95 of taxable income. The explanation provides no credits or rate computation capable of reducing the liability to its submitted amount."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It gave no usable derivation and submitted an amount consistent with omitting the $25,000 qualified-overtime deduction. The correct deduction stack leaves $188,883.95 taxable, including $6,208 taxed at qualified-dividend rates."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It wrongly excluded the separately listed $16,408 ESI premium from the wage base and also deducted a $580 charitable amount on top of the standard deduction. Those errors reduced taxable income to $173,197 instead of $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It deducted cash charity and vehicle-loan interest in addition to the standard deduction, even though neither extra deduction appears in the traced $57,779.77 deduction total. The computation should use the $32,200 standard deduction and $25,000 overtime deduction, with only the remaining traced adjustment bringing taxable income to $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It wrongly excluded the $16,408 ESI premium from wages and deducted $580 of cash charity alongside the standard deduction. This understated taxable income at $173,197 rather than the traced $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated vehicle-loan interest as an allowable additional deduction and did not reproduce the traced deduction total or taxable income. The applicable deductions reduce $246,663.72 of AGI to $188,883.95, whose ordinary and qualified-dividend taxes total $30,543.91."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It used roughly $263,000 of AGI and only about $20,000 of deductions, ignoring both the traditional 401(k) wage exclusion and the $25,000 overtime deduction. The traced AGI is $246,663.72 and total deductions are $57,779.77."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It assumed a TCJA sunset, itemized $26,736, and claimed personal exemptions, while omitting the $25,000 qualified-overtime deduction. The applicable deduction method is the $32,200 standard deduction plus the overtime deduction, producing $188,883.95 of taxable income."
-us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its explanation relies on itemized mortgage, property-tax, and charitable deductions and never applies the $25,000 qualified-overtime deduction. The household instead uses the $32,200 standard deduction, and total traced deductions reduce taxable income to $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It understated AGI through an unsupported wage adjustment and then omitted the $25,000 qualified-overtime deduction, using only the $32,200 standard deduction. The traced figures are $246,663.72 of AGI and $188,883.95 of taxable income."
-us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"It wrongly excluded the $16,408 ESI premium from gross wages and assumed expired-TCJA itemization, personal exemptions, and pre-TCJA rates. It also omitted the $25,000 qualified-overtime deduction used in the 2026 computation."
-us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It used AGI of $247,385 instead of $246,663.72 by denying the traced $721 IRA deduction, then added a $580 nonitemizer charitable deduction that is absent from the traced deduction total. The net result overstated taxable income by $721 at $189,605 instead of $188,883.95."
-us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It treated the overtime deduction as $25,675 rather than the $25,000 joint-filer cap and used an estimated $30,900 standard deduction instead of $32,200. It also placed the overtime deduction directly in its AGI calculation rather than following the traced AGI-to-taxable-income computation."
-us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted the $15,436 401(k) contribution incorrectly, reducing $262,821 only to $258,821, and then claimed $15,395 of itemized deductions exceeded a $32,000 standard deduction. It omitted the $25,000 overtime deduction and submitted $51,214.75 despite its own recomputed regular tax being $42,933.38."
-us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It omitted the $25,000 qualified-overtime deduction, used an incorrect $32,900 standard deduction, and treated nondeductible Roth contributions as reductions in taxable income. It then severely misapplied the married-joint brackets, assigning $44,586.80 of ordinary tax to $205,441 of ordinary income."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The reasoning reached the correct $25,000 overtime deduction but used estimated bracket thresholds and taxable income of $190,185 instead of $188,883.95. It then submitted $33,223 despite its own calculation yielding $30,830, so the final value also contradicts its stated arithmetic."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified-overtime deduction and used a $29,200 standard deduction instead of $32,200. It also treated the $16,408 employer-sponsored premium as an additional wage reduction even though the engine’s employment-income construction already yields $237,564 after the traditional 401(k) contribution."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified-overtime deduction and used an estimated $31,500 standard deduction, leaving taxable income far above $188,883.95. It also improperly added Additional Medicare Tax to an individual-income-tax output that excludes payroll taxes, then submitted $49,861 rather than either tax total shown in its reasoning."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It omitted the $25,000 qualified-overtime deduction and used an estimated $30,300 standard deduction instead of $32,200. Its submitted $53,503 also contradicts its own stated total of approximately $37,055."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It recognized the expanded 2026 SALT cap but neither applied the $25,000 overtime deduction nor calculated the rate schedule correctly. Taxable income after the controlling standard and overtime deductions is $188,883.95, and the stated roughly $201,300 base cannot produce $47,500 under the joint brackets."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It completely omitted the $25,000 qualified-overtime deduction and used a $30,000 standard deduction instead of $32,200. Those errors raised taxable income from $188,883.95 to $217,385 and drove the excess tax."
+us,scenario_049,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omitted the capped $25,000 qualified-overtime deduction despite the listed $44,647 overtime premium. It then appended an unexplained increase from its own $38,831 calculation to $62,700, including payroll-tax concepts that do not belong in this output."
+us,scenario_049,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied a nonexistent 2026 TCJA sunset, restoring personal exemptions and the old 10/15/25/28-percent schedule instead of the applicable 2026 law. It also subtracted the $16,408 employer premium from wages and omitted the $25,000 overtime deduction and $32,200 standard deduction used in the reference computation."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It incorrectly treated 2026 as a post-TCJA-expiration year, claimed personal exemptions, and used obsolete ordinary brackets. It also separately excluded the $16,408 employer premium from wages and omitted the $25,000 qualified-overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its approximate $229,000 taxable-income base fails to apply the deductions that reduce taxable income to $188,883.95, most importantly the $32,200 standard deduction and $25,000 overtime deduction. The resulting $52,367.59 is also incompatible with the applicable joint rate schedule."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It incorrectly applied post-TCJA rules, including personal exemptions and pre-TCJA itemized-deduction treatment. It also excluded the $16,408 employer premium from wages and failed to apply the $25,000 qualified-overtime deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"It used obsolete post-TCJA-expiration brackets and personal exemptions to derive $194,141 of taxable income. The applicable computation instead uses the $32,200 standard deduction and $25,000 overtime deduction to reach $188,883.95."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It incorrectly reduced wages by the separately listed $16,408 employer premium and used a $30,000 standard deduction. It omitted the $25,000 qualified-overtime deduction, so its $200,977 taxable-income figure does not follow the applicable deduction stack."
+us,scenario_049,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer implies a taxable-income base materially above $188,883.95 and does not account for the capped $25,000 qualified-overtime deduction. Applying that deduction with the $32,200 standard deduction yields $29,612.71 of ordinary tax plus $931.20 on qualified dividends."
+us,scenario_049,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. This is a missing-output contract failure rather than a substantive tax computation.
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"Its explanation never identifies the $25,000 overtime deduction or a reproducible taxable-income calculation, and $20,460 is far below the tax generated by $188,883.95 of taxable income. The number reflects a misapplication of the joint brackets or subtraction of deductions or credits that do not apply."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The answer gives no usable derivation and exceeds the tax on the correct $188,883.95 taxable-income base by $8,022.09. Its amount is consistent with omitting the $25,000 qualified-overtime deduction and taxing a substantially larger base."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It double-counted the $16,408 employer health premium as a wage exclusion, reducing AGI to $230,977 even though the engine’s employment income is $237,564 after the traditional 401(k) contribution. That erroneous exclusion drove taxable income down to $173,197 instead of $188,883.95."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It improperly included a vehicle-loan-interest deduction even though the income phaseout eliminates it for this joint filer, and its deduction stack did not reproduce $188,883.95 of taxable income. The ordinary schedule on the correct base plus $931.20 of qualified-dividend tax yields $30,543.91, not $30,498.38."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"It separately excluded the $16,408 employer health premium from wages, producing $230,977 of AGI and $173,197 of taxable income. The correct AGI is $246,663.72 and the standard-plus-overtime deduction stack produces $188,883.95."
+us,scenario_049,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It treated vehicle-loan interest as deductible despite the applicable joint-income phaseout and did not reproduce the engine’s taxable income of $188,883.95. That incorrect deduction and base calculation caused the $2,929.09 overstatement."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It used roughly $263,000 of AGI and roughly $230,000 of taxable income, failing to subtract the traditional 401(k) contribution and the $25,000 qualified-overtime deduction correctly. The applicable deduction sequence produces $188,883.95 of taxable income."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It incorrectly assumed a 2026 TCJA sunset, claimed personal exemptions, and used the restored pre-TCJA rate structure. It also omitted the $25,000 qualified-overtime deduction that is central to the correct taxable-income calculation."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It applied obsolete post-sunset personal exemptions and the 10/15/25-percent rate schedule. It also omitted the $25,000 qualified-overtime deduction, leaving taxable income at $209,739 rather than $188,883.95."
+us,scenario_049,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"Its generic itemized-deduction approach does not apply the controlling $32,200 standard deduction together with the $25,000 qualified-overtime deduction. The resulting answer reflects a taxable-income base above $188,883.95."
+us,scenario_049,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It used AGI of $230,256, reflecting an unwarranted exclusion beyond the traditional 401(k), and omitted the $25,000 qualified-overtime deduction. Those errors produced $198,056 of taxable income instead of $188,883.95."
+us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"It incorrectly treated the $16,408 employer premium as an additional wage exclusion and applied expired-law personal exemptions and pre-TCJA brackets. It also omitted the $25,000 qualified-overtime deduction required under the applicable 2026 rules."
+us,scenario_049,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It correctly identified the standard and overtime deductions but treated only the $580 cash gift as an extra deduction and arrived at $189,605 rather than $188,883.95. The missing $721.05 reduction corresponds to the deductible traditional IRA contribution, which lowers AGI to $246,663.72."
+us,scenario_049,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It incorrectly placed a $25,675 overtime amount above the line, deducted the traditional IRA without reconciling the resulting AGI, and used an estimated $30,900 standard deduction. The applicable figures are AGI of $246,663.72, a capped $25,000 overtime deduction, and a $32,200 standard deduction."
+us,scenario_049,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It correctly deducted the traditional IRA but completely omitted the $25,000 qualified-overtime deduction. This left taxable income at $214,464 rather than $188,883.95 and overstated the tax accordingly."
+us,scenario_049,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It subtracted only $4,000 of the stated $15,436 traditional 401(k) contribution, incorrectly claimed $15,395 of itemized deductions exceeded a roughly $32,000 standard deduction, and omitted the $25,000 overtime deduction. It also taxed all $6,320 of dividends preferentially even though only $6,208 are qualified, then submitted $51,214.75 despite deriving $42,933.38."
+us,scenario_049,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"It deducted the Roth 401(k) contribution as though it reduced taxable income and used $32,900 rather than the $32,200 standard deduction, while omitting the $25,000 overtime deduction. It then computed $44,586.80 of ordinary tax on $205,441, which is not the result of the applicable married-joint rate schedule."
 us,scenario_049,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_049,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_049,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -3096,30 +3256,31 @@ us,scenario_049,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_ou
 us,scenario_049,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model's reasoning correctly determined that the 53-year-old head lacks both the age threshold and any qualifying early-Medicare condition, but it submitted value 1 instead of 0. Its numeric output directly contradicts its own eligibility analysis and required final marker."
 us,scenario_049,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_049,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_049,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model correctly derived $11,439 of Social Security tax, $3,668.50 of Medicare tax, and $27 of Additional Medicare Tax, but submitted $14,503.98 instead of their $15,134.50 sum. Its final value does not implement its own payroll-tax computation."
-us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the obsolete $168,600 Social Security wage base instead of $184,500 and applied the $200,000 Additional Medicare threshold instead of the $250,000 joint threshold. It then invented a $904.13 New Hampshire adjustment despite acknowledging that New Hampshire has no employee payroll tax, and even those stated components do not sum to $20,502.83."
-us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model capped Social Security wages at $176,100 instead of the 2026 wage base of $184,500. This understated Social Security tax by $520.80 while its Medicare and $27 Additional Medicare Tax calculations were correct."
-us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model explicitly calculated the correct three components and their $15,134.50 total, then submitted $13,164.85. The submitted value contradicts its own completed computation."
-us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model ultimately substituted the 2025 Social Security wage base of $176,100 for the 2026 base of $184,500. That reduced employee Social Security tax from $11,439 to $10,918.20 and understated the total by $520.80."
-us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model used a $176,100 Social Security wage base rather than the 2026 $184,500 base. Its Medicare and Additional Medicare Tax calculations were correct, leaving the entire $520.80 error in Social Security tax."
-us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted $16,408 of employer-sponsored insurance premiums from FICA wages and also used a $176,100 Social Security cap instead of $184,500. The premium amount is not identified as a pre-tax employee salary reduction, so Medicare applies to all $253,000 and the resulting wages also generate $27 of Additional Medicare Tax."
-us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly treated the $16,408 employer-sponsored insurance premium as a pre-tax reduction of Social Security and Medicare wages and therefore omitted the $27 Additional Medicare Tax. It also used an estimated $183,900 Social Security base instead of $184,500."
-us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model named a $176,100 Social Security wage base instead of the 2026 $184,500 base. Moreover, 6.2% of $176,100 plus 1.45% of $253,000 and 0.9% of the $3,000 excess equals $14,613.70, not its submitted $13,959.81."
-us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,408 employer-sponsored insurance premium from FICA wages. Medicare tax applies to the full $253,000, the Social Security cap is $184,500, and wages above the joint $250,000 threshold add $27 of Additional Medicare Tax."
-us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model used an incorrect $174,900 Social Security wage base and improperly reduced Medicare wages to $236,592 by subtracting employer-sponsored insurance premiums. It consequently also omitted the $27 Additional Medicare Tax due on $3,000 above the joint threshold."
-us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $16,408 employer-sponsored insurance premium and used a $181,200 Social Security cap instead of $184,500. This also caused it to omit the $27 Additional Medicare Tax on wages exceeding $250,000."
-us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The stated formula yields $15,134.50 when the 2026 $184,500 Social Security wage base, full $253,000 Medicare wage base, and $3,000 Additional Medicare excess are used. Its $14,613.34 answer instead reflects an obsolete Social Security cap and is also $0.36 below the corresponding $14,613.70 component sum."
-us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, so the required numeric value was missing."
-us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The correct components under the formula it stated are $11,439 of Social Security tax, $3,668.50 of Medicare tax, and $27 of Additional Medicare Tax. Its $19,648 submission exceeds that $15,134.50 sum by $4,513.50 and therefore applies an unsupported payroll-tax base, rate, or extra component."
-us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's stated components produce $15,134.50 using the $184,500 Social Security wage base, 1.45% Medicare tax on $253,000, and $27 Additional Medicare Tax. Its unexplained $15,302 estimate overstates that calculation by $167.50."
-us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model improperly excluded $16,408 of employer-sponsored insurance premiums from FICA wages, used a $183,600 Social Security cap instead of $184,500, and omitted Additional Medicare Tax. Medicare applies to the full $253,000, and $3,000 above the joint threshold produces an additional $27."
-us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model correctly used the $184,500 Social Security cap but improperly subtracted $16,408 of employer-sponsored insurance premiums from Medicare wages. Medicare applies to all $253,000, and the resulting $3,000 excess over the joint threshold also generates $27 of Additional Medicare Tax."
-us,scenario_049,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model rounded the Social Security base to roughly $180,000 and then rounded the entire liability to $14,000 instead of using the exact statutory inputs. Applying the $184,500 cap and retaining the exact $3,668.50 Medicare and $27 Additional Medicare amounts yields $15,134.50."
-us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model omitted the $27 Additional Medicare Tax even though the spouse's $253,000 of wages exceeds the $250,000 joint threshold by $3,000. Its remaining $14,808 figure also understates the $15,107.50 sum of Social Security tax at the $184,500 cap and standard Medicare tax on all wages."
-us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model improperly deducted the $16,408 employer-sponsored insurance premium from Medicare wages. Medicare applies to the full $253,000, and those wages exceed the $250,000 joint threshold, producing $27 of Additional Medicare Tax."
-us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model improperly reduced both Social Security and Medicare wages by the $16,408 employer-sponsored insurance premium and used an estimated $183,000 Social Security cap instead of $184,500. It then omitted the $27 Additional Medicare Tax because of that unsupported wage reduction."
-us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model vacillated among obsolete Social Security wage bases but never carried the correct $184,500 base into its submission. Worse, none of its displayed totals exceeds $14,613.70, so its submitted $19,354.50 contradicts every computation in its reasoning."
-us,scenario_049,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model overstated Social Security tax by using $11,475 instead of $11,439, then made a major addition error: $11,475 plus $3,668.50 plus $27 equals $15,170.50, not $23,780.75. Using the correct Social Security component produces $15,134.50."
+us,scenario_049,payroll_tax,claude-fable-5,llm_error,other,False,"The model's reasoning computed all three components correctly and explicitly totaled them to $15,134.50, but it submitted $14,503.98. This is a final-answer transcription or arithmetic-contract error."
+us,scenario_049,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used the obsolete $168,600 Social Security wage base instead of $184,500 and applied Additional Medicare Tax above $200,000 rather than the $250,000 joint threshold. It then invented a $904.13 New Hampshire payroll-tax adjustment despite stating that New Hampshire imposes no employee payroll tax."
+us,scenario_049,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a projected $176,100 Social Security wage base instead of the 2026 base of $184,500. That understated employee Social Security tax by $520.80."
+us,scenario_049,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly derived $11,439 of Social Security tax, $3,668.50 of Medicare tax, and $27 of Additional Medicare Tax, then correctly totaled $15,134.50. Its submitted $13,164.85 is an unsupported final-answer transcription error."
+us,scenario_049,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model ultimately substituted the 2025 Social Security wage base of $176,100 for the 2026 base of $184,500. This reduced Social Security tax from $11,439 to $10,918.20 and understated the total by $520.80."
+us,scenario_049,payroll_tax,claude-sonnet-5,llm_error,thresholds_rates,False,"The model applied the 6.2% Social Security rate to the 2025 wage base of $176,100 instead of the 2026 wage base of $184,500. Its Medicare and Additional Medicare Tax calculations were otherwise correct."
+us,scenario_049,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the listed $16,408 employer-sponsored insurance premiums as a pre-tax employee payroll deduction and reduced Medicare wages to $236,592, eliminating the $27 Additional Medicare Tax. It also used the $176,100 Social Security cap instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,408 employer-sponsored insurance premium from FICA wages, reducing Medicare wages to $236,592 and eliminating Additional Medicare Tax. It also used an $183,900 Social Security wage base rather than $184,500."
+us,scenario_049,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The stated method used the $176,100 Social Security wage base instead of $184,500, and the submitted amount does not equal even the components it described. Correct application of those components yields $15,134.50."
+us,scenario_049,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $16,408 employer-sponsored insurance premium and omitted the Additional Medicare Tax. Payroll Medicare wages remain $253,000, producing $3,668.50 of regular Medicare tax and $27 of Additional Medicare Tax."
+us,scenario_049,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages to $236,592 by subtracting employer-sponsored insurance premiums and omitted the $27 Additional Medicare Tax. It also used a projected $174,900 Social Security wage base instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,408 employer-sponsored insurance premium from Medicare wages and consequently omitted Additional Medicare Tax. It also capped Social Security wages at $181,200 instead of $184,500."
+us,scenario_049,payroll_tax,gemini-3.7-flash,llm_error,other,False,"The model named the correct tax components but submitted $14,613.34, a figure consistent with using roughly the prior-year Social Security cap rather than the $184,500 cap. Applying the 2026 cap and the stated Medicare rules yields $15,134.50."
+us,scenario_049,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no payroll-tax output or explanation, so it failed the required output contract."
+us,scenario_049,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model described the correct three-component structure but submitted $19,648, which does not follow from the applicable wage base, rates, or Additional Medicare Tax threshold. Those components are $11,439, $3,668.50, and $27, totaling $15,134.50."
+us,scenario_049,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly said employer-sponsored insurance does not reduce payroll wages but submitted an unsupported estimate of $15,302. Applying the $184,500 Social Security cap and $250,000 Additional Medicare Tax threshold yields $15,134.50."
+us,scenario_049,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model improperly excluded the $16,408 employer-sponsored insurance premium from FICA wages, reducing regular Medicare tax and eliminating Additional Medicare Tax. It also used a Social Security wage base of $183,600 instead of $184,500."
+us,scenario_049,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model correctly calculated Social Security tax using the $184,500 cap but improperly reduced Medicare wages by the $16,408 employer-sponsored insurance premium. Medicare tax applies to $253,000, and the $3,000 excess over the joint threshold also produces $27 of Additional Medicare Tax."
+us,scenario_049,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the Social Security wage base to approximately $180,000 and then rounded the entire result to $14,000. Using the exact $184,500 cap and retaining the precisely calculated Medicare components gives $15,134.50."
+us,scenario_049,payroll_tax,grok-4.6,llm_error,thresholds_rates,False,"The model used an approximate $183,600 Social Security wage base instead of $184,500. That reduced Social Security tax by $55.80, accounting for its rounded $15,079 result rather than $15,134.50."
+us,scenario_049,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model explicitly omitted Additional Medicare Tax even though $253,000 of Medicare wages exceeds the $250,000 joint threshold by $3,000. It also failed to apply the exact $184,500 Social Security wage base, leaving its $14,808 total unsupported."
+us,scenario_049,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly used the $184,500 Social Security cap but improperly subtracted $16,408 of employer-sponsored insurance premiums from Medicare wages. Using the full $253,000 produces $3,668.50 of regular Medicare tax plus $27 of Additional Medicare Tax."
+us,scenario_049,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model improperly subtracted the employer-sponsored insurance premium from both Social Security and Medicare wages and therefore eliminated Additional Medicare Tax. It also used an estimated $183,000 Social Security cap instead of $184,500."
+us,scenario_049,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model's submitted $19,354.50 matches none of its several alternative calculations and contradicts its own final component total. The correct chosen inputs are the $184,500 Social Security cap, $253,000 of Medicare wages, and $3,000 of wages subject to Additional Medicare Tax, totaling $15,134.50."
+us,scenario_049,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated components of $11,475, $3,668.50, and $27 but added them incorrectly to $23,780.75; they total $15,170.50. Its Social Security component was also $36 too high because the applicable amount is 6.2% of $184,500, or $11,439, producing the final $15,134.50 total."
 us,scenario_049,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_049,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_049,snap,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -3138,32 +3299,34 @@ us,scenario_049,state_income_tax_before_refundable_credits,inkling,llm_error,thr
 us,scenario_049,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly derived $5,021 after the joint exemption but treated that balance as subject to the 1% rate automatically. New Hampshire's filing requirement threshold applies before the rate, and the $5,021 balance is below it, yielding $0."
 us,scenario_049,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_049,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model added the $180 non-Schedule-D capital-gain input to the $40,000 IRS gross income, producing $24,080 instead of $23,900 of taxable income. Its stated bracket calculation yields $2,641.60, but it then submitted $3,086 without any additional tax or credit supporting that increase."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated the employer-sponsored insurance premium as a deduction from the listed gross wages and used the wrong standard deduction. It then reduced pre-refundable-credit income tax to zero using EITC and withholding, even though EITC is refundable, withholding is a payment rather than a tax reduction, and neither belongs in this output."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model added the $180 input to IRS gross income, yielding $24,080 rather than $23,900 of taxable income. Its own bracket arithmetic produces about $2,642, after which it submitted $2,960 based on an unsupported switch to different parameters."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model added the $180 input to the $40,000 IRS gross-income base and calculated about $2,642 on $24,080 of taxable income. It then submitted $2,867 despite identifying no supplemental tax or nonrefundable-credit adjustment that changes its calculation."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model improperly included the $180 input in IRS gross income, so it used $24,080 rather than $23,900 of taxable income. Its displayed rate calculation is about $2,642, but the submitted $2,955 does not follow from that arithmetic."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the 2025 $15,000 standard deduction and 2025 bracket threshold instead of the 2026 $16,100 deduction and 2026 rates, while also adding the $180 input to IRS gross income. Those choices raised taxable income to $25,180 rather than $23,900 and produced $2,783 instead of $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model added the $180 input to IRS gross income and initially calculated roughly $2,647 on $24,080 of taxable income. It then increased the answer to $3,059 through unspecified “adjustments and rounding,” even though rounding cannot create the $412 increase and no additional tax applies."
-us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model subtracted the $5,389 employer-sponsored insurance premium from wages and applied a post-TCJA-expiration regime with a personal exemption, smaller standard deduction, and 15% bracket. The applicable 2026 computation instead uses $40,000 of gross income, a $16,100 standard deduction, and the 10% and 12% rates."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model deducted the employer-sponsored insurance premium from already stated gross wages and then applied an expired-law personal exemption, reduced standard deduction, and 15% bracket. The correct computation applies the $16,100 standard deduction to $40,000 and taxes $23,900 under the 2026 10% and 12% brackets."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model invented a $1,300 nonrefundable child or other credit for a single filer with no dependents and no qualifying credit facts. It also used a $15,000 deduction and included the $180 input rather than deriving $23,900 of taxable income from $40,000 less the $16,100 standard deduction."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model improperly deducted the employer-sponsored insurance premium from gross wages and applied an expired-law combination of a smaller standard deduction and personal exemption. The 2026 computation has $40,000 of IRS gross income, a $16,100 standard deduction, $23,900 of taxable income, and $2,620 of tax."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model reduced wages by the employer-sponsored insurance premium and used an expired-law personal exemption, smaller standard deduction, and 15% marginal bracket. Applying the 2026 $16,100 standard deduction and 10%/12% rates to the $40,000 gross-income base yields $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $5,389 employer-sponsored insurance premium from the listed gross wages and added the $180 input, reducing its AGI to $34,791. It also used a $15,700 rather than $16,100 standard deduction, producing $19,091 instead of $23,900 of taxable income."
-us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied an unspecified standard-deduction-plus-personal-exemption regime and a 15% bracket instead of the applicable $16,100 standard deduction and 10%/12% rate schedule. It also included the $180 input in AGI, whereas the traced IRS gross-income base is $40,000."
-us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 standard deduction and included the $180 input, producing $25,580 instead of $23,900 of taxable income. Its displayed tax is $2,831.10, but it submitted $2,530 without identifying any nonrefundable credit, and its premium-tax-credit discussion is irrelevant because the ACA credit is excluded from this output."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as eliminating all tax even though $40,000 less the $16,100 deduction leaves $23,900 of taxable income. With no nonrefundable credits, applying the 2026 rates to that amount yields $2,620 rather than zero."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model asserted unspecified “nonrefundable taxes reduced for the modeled situation” without naming any available credit or showing a calculation. The actual sequence is $40,000 minus the $16,100 standard deduction, followed by the 2026 rates on $23,900, with no nonrefundable credits, yielding $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $15,350 standard deduction and $12,000 bracket threshold instead of the applicable 2026 $16,100 deduction and rate thresholds. It also added the $180 input to gross income, so its taxable-income base was $24,830 rather than $23,900."
-us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model added the $180 non-Schedule-D capital-gain input to the traced $40,000 IRS gross-income base. That produced $24,080 of taxable income and $2,641.60 of tax instead of applying the $16,100 deduction to $40,000 and obtaining $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used approximately $24,680 of taxable income instead of the traced $23,900 produced by $40,000 less the $16,100 standard deduction. With no nonrefundable credits or supplemental taxes, the 2026 rates yield $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the 2025 $15,000 standard deduction and 2025 bracket threshold and included the $180 input in AGI. The 2026 computation instead subtracts $16,100 from $40,000, leaving $23,900 and producing $2,620."
-us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model deducted the employer-sponsored insurance premium from the stated wages and applied an expired-law personal exemption, smaller standard deduction, and 15% bracket. The applicable 2026 rules use $40,000 of IRS gross income, a $16,100 standard deduction, and the 10%/12% schedule."
-us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required structured-output contract."
-us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model included the $180 input in taxable income and used the wrong 10% bracket endpoint. It then invented a large “0% LTCG adjustment” even though only $180 was listed and a preferential capital-gain rate is a rate computation, not a credit capable of reducing $2,635 to $2,090."
-us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own calculation produced $2,618.60, essentially the tax from its chosen deduction and brackets, but it submitted $1,410 with no credit or tax adjustment supporting the reduction. It also included the $180 input and used a $16,300 deduction rather than deriving $23,900 from $40,000 less $16,100."
-us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model asserted that nonrefundable credits reduce the liability to zero even though the household has no dependents or other facts establishing such a credit. The $16,100 standard deduction leaves $23,900 taxable, and the 2026 rates produce $2,620 with no nonrefundable-credit reduction."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's stated calculation produces $2,641.60, but it submitted $3,086 without any intervening tax, credit, or adjustment. It also included the $180 capital-gain input that the traced gross-income computation excludes; using $40,000 minus $16,100 produces $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly reduced this before-refundable-credit liability to zero using EITC and withholding. EITC is refundable and withholding is a payment, so neither reduces federal income tax before refundable credits; the ordinary tax computation yields $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's displayed bracket calculation produces approximately $2,642, but it submitted $2,960 after invoking unspecified alternative inflation adjustments. It also included the $180 capital-gain input instead of using the traced $40,000 gross income."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model calculated $2,642 from its own inputs and then submitted $2,867 without a corresponding computation. The traced calculation excludes the $180 from gross income and taxes $23,900, producing $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The stated arithmetic gives about $2,642, not the submitted $2,955. The model also added the $180 to gross income, whereas the traced tax base is $40,000 less the $16,100 standard deduction."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $15,000 standard deduction as a proxy instead of the 2026 $16,100 amount and included the $180 in gross income. Those choices produced $25,180 of taxable income rather than the traced $23,900."
+us,scenario_051,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's explicit bracket arithmetic equals $2,646.60, yet it submitted $3,059 based on nonexistent rounding and unspecified standard-deduction adjustments. It also included the $180 that is absent from the traced IRS gross-income base."
+us,scenario_051,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed expiration of the applicable tax regime and used a personal exemption, a $7,493 standard deduction, and a 15% second bracket. The 2026 computation instead uses a $16,100 standard deduction, no personal exemption, and 10%/12% rates on $23,900."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied obsolete post-TCJA-expiration parameters: an $8,300 standard deduction, a personal exemption, and a 15% second rate. It also subtracted employer-sponsored insurance premiums from wages even though the traced gross-income amount is $40,000."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model invented a $1,300 nonrefundable child or other credit for a single taxpayer with no dependents and no stated credit-qualifying facts. No nonrefundable credit applies, and the tax after the $16,100 standard deduction is $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly deducted the $5,389 employer-sponsored insurance amount from reported wages and applied a standard-deduction-plus-personal-exemption regime. The traced computation uses $40,000 of gross income, a $16,100 standard deduction, and no personal exemption."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an obsolete $8,300 standard deduction, personal exemption, and 15% second bracket while also subtracting employer-sponsored insurance premiums from wages. The applicable computation uses $23,900 of taxable income under the 10% and 12% brackets."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $5,389 of employer-sponsored insurance premiums from the reported $40,000 wages and used a $15,700 rather than $16,100 standard deduction. The traced taxable income is $23,900, not $19,091."
+us,scenario_051,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model invoked a personal exemption and a 15% bracket that do not apply in the traced 2026 computation. The liability is calculated with only the $16,100 standard deduction and the 10% and 12% brackets."
+us,scenario_051,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model used an incorrect $14,600 standard deduction and calculated $2,831.10, then submitted $2,530 without identifying any valid reduction. Premium Tax Credit reasoning is irrelevant because that credit is excluded from this output and cannot reconcile the submitted amount."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the standard deduction as eliminating all taxable income even though $40,000 minus $16,100 leaves $23,900. With no nonrefundable credits, the resulting $2,620 tax is not reduced to zero."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model supplied no concrete deduction, bracket, or nonrefundable-credit calculation supporting $2,146 and referred vaguely to unspecified reductions. The specified computation leaves $23,900 taxable and produces $2,620 with no nonrefundable credits."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,350 standard deduction and included the $180 in its income base. The applicable $16,100 deduction against $40,000 produces $23,900 of taxable income and $2,620 of tax."
+us,scenario_051,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model added the $180 non-Schedule-D capital-gain input to gross income, raising taxable income from $23,900 to $24,080. Removing that amount from the traced gross-income base lowers the 12%-bracket tax by $21.60, from $2,641.60 to $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used approximately $24,680 of taxable income rather than the traced $23,900. Applying the $16,100 standard deduction to $40,000 and then the applicable brackets yields $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used the 2025 $15,000 standard deduction and included the $180 in gross income, producing taxable income of $25,180. The 2026 deduction is $16,100 and the traced gross-income base is $40,000, leaving $23,900."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,750 standard deduction and included the $180 in gross income. The traced calculation instead subtracts $16,100 from $40,000, reducing taxable income to $23,900."
+us,scenario_051,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly reverted to pre-TCJA rules, using a personal exemption, a small standard deduction, and a 15% second bracket. It also deducted employer-sponsored insurance premiums from wages, contrary to the traced $40,000 gross-income base."
+us,scenario_051,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. This is a missing-output contract failure rather than a substantive tax calculation.
+us,scenario_051,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,The model invented a large nonrefundable capital-gains adjustment after already treating the $180 as ordinary income. A 0% long-term-capital-gains rate is not a credit and cannot reduce ordinary tax by $545; no nonrefundable credit applies.
+us,scenario_051,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model added the $180 non-Schedule-D capital-gain input to the traced gross-income base. Using $40,000 rather than $40,180 leaves $23,900 taxable and reduces its result by $21.60 to $2,620."
+us,scenario_051,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own bracket calculation equals $2,618.60, but it submitted $1,410 with no stated adjustment. It also used a $16,300 deduction and included the $180, rather than applying the traced $16,100 deduction to $40,000."
+us,scenario_051,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model asserted that unspecified nonrefundable credits reduce the liability to zero, but this household has no qualifying nonrefundable credit. The $23,900 taxable-income calculation therefore remains at $2,620 before refundable credits."
 us,scenario_051,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model awarded $560 of childless EITC without applying the phaseout endpoint for a single filer. At $40,000 of earnings, the childless EITC has fully phased out."
 us,scenario_051,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required output contract."
 us,scenario_051,federal_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own EITC calculation correctly reached $0 and stated that no other refundable credits apply, but it submitted $2,557.60. The submitted value contradicts its completed derivation and does not represent any applicable refundable-credit component."
@@ -3181,120 +3344,126 @@ us,scenario_051,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
 us,scenario_051,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll_tax value or explanation. It therefore failed the required structured-output contract.
 us,scenario_051,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model correctly derived $2,480 of Social Security tax plus $580 of Medicare tax and explicitly totaled them as $3,060, but submitted $3,074.95. The submitted value is inconsistent with every component in its explanation."
 us,scenario_051,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $180 capital-gains input to Louisiana AGI and used a $12,500 deduction instead of $12,835. Its stated calculation produces $830.40, but it then submitted an unsupported $1,020 that contradicts its own arithmetic."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely treated Louisiana as having no individual income tax. Louisiana applies a 3% tax to $27,165 of taxable income, producing $814.95."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly increased AGI to $40,180 and used a $12,500 standard deduction. The applicable inputs are $40,000 of AGI and a $12,835 deduction, leaving $27,165 taxable."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used the wrong AGI and standard deduction, then invented a personal credit that reduced its own $830.40 calculation to $632. No such reduction applies in this calculation."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of AGI and a $12,500 deduction rather than $40,000 and $12,835. It then replaced its resulting tax of about $830 with an unsupported rounding adjustment to $812."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted estimated federal income tax and used a $12,500 Louisiana standard deduction. Louisiana taxable income here is simply $40,000 minus $12,835, so the extra federal-tax deduction drove its answer too low."
-us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $40,180 of AGI and a $12,500 deduction, then submitted $1,080 despite explicitly calculating $830.40. Its final value has no supporting computation and omits the applicable $12,835 deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete 4.25% rate and a $4,500 exemption to an unsupported $34,791 AGI. The applicable calculation uses a 3% rate, $40,000 AGI, and a $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an unsupported $34,791 AGI, deducted federal tax, and applied an obsolete progressive rate schedule with a $4,500 deduction. The 2026 calculation instead applies a flat 3% rate after the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $724 implies the model used the wrong progressive schedule or taxable-income base. The required computation is 3% of $27,165 after subtracting $12,835 from $40,000."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an unsupported $34,791 AGI, a $4,500 exemption, and obsolete 1.85% and 3.5% brackets. Louisiana's 2026 flat 3% rate applies to $27,165 after the $12,835 deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model deducted federal income tax, used only a $4,500 state deduction, and applied the obsolete progressive schedule. The correct base is $40,000 minus $12,835, taxed at a flat 3%."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an unsupported $34,791 AGI, a $4,500 exemption, and obsolete progressive rates. It should have applied 3% to $27,165 after the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model added $180 to AGI, allowed only a $4,500 deduction, and applied obsolete 1.85% and 3.5% brackets. The applicable base is $27,165 under the flat 3% regime."
-us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $180 capital-gains input to AGI and used a $12,500 deduction. PolicyEngine's state calculation uses $40,000 of AGI less the $12,835 standard deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions or exemptions eliminated the liability. The $12,835 standard deduction leaves $27,165 taxable, so the tax is not zero."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The unexplained $781 does not apply the traced taxable-income calculation. Subtracting $12,835 from $40,000 leaves $27,165, and 3% of that amount is $814.95."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $40,180 of AGI and a $12,500 standard deduction. The applicable figures are $40,000 and $12,835, yielding $27,165 taxable income."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $180 capital-gains input to AGI and used a $12,500 deduction. The state tax base is $40,000 minus $12,835."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of income and a $12,500 deduction rather than the traced $40,000 AGI and $12,835 deduction. This overstated taxable income by $515."
-us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model computed tax on $27,680 by adding the capital-gains input to AGI and using a $12,500 deduction. The correct taxable income is $27,165."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model supplied $680 without identifying or applying the governing deduction and rate. The required derivation subtracts $12,835 from $40,000 and taxes the remaining $27,165 at 3%."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted the federal standard deduction for Louisiana's $12,835 standard deduction and then applied obsolete progressive rates. Louisiana taxable income is $27,165 and is taxed at a flat 3%."
-us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model started from an unsupported $34,791 AGI and improperly deducted federal tax plus two separate $4,500 allowances. It also used obsolete progressive brackets instead of applying 3% to $27,165."
-us,scenario_051,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $40,180 of AGI and a $12,500 deduction, producing $830.40. It then rounded away cents even though the requested annual amount and its own explanation supported an exact monetary value."
-us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the required submission contract."
-us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $180 capital-gains input to AGI and used a $12,500 deduction. The traced calculation uses $40,000 of AGI and a $12,835 deduction."
-us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used only a $4,500 deduction instead of $12,835 and contradicted its own $1,065 calculation by submitting $1,208. Its discussion of eliminating Louisiana income tax was also inconsistent with applying the 3% tax."
-us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model improperly used an estimated federal standard deduction, added a separate $4,500 exemption, and applied obsolete progressive rates. It then submitted $573 despite its own arithmetic producing $482.52; the governing calculation is 3% of $27,165."
-us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model's unsupported $1,324.80 does not subtract the applicable $12,835 standard deduction from $40,000 of AGI. Applying the 3% rate to the resulting $27,165 yields $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model added the $180 capital-gains input to Louisiana AGI and used a $12,500 deduction instead of the engine's $40,000 AGI and $12,835 deduction. Its submitted $1,020 also contradicts its own computed $830.40 and has no valid computation behind it."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely treated Louisiana as a state without an individual income tax. Louisiana applies a 3% tax to the $27,165 taxable base, producing $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used AGI of $40,180 and a $12,500 standard deduction. The required inputs are $40,000 of AGI and a $12,835 Louisiana standard deduction, leaving $27,165 taxable and $814.95 of tax."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"After using the wrong AGI and deduction, the model invented an unspecified personal credit that reduced its own $830.40 calculation to $632. No such reduction applies in this computation; 3% of $27,165 is $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of AGI and a $12,500 deduction, then replaced its resulting $830.40 with an unsupported rounding adjustment to $812. The engine uses $40,000 less $12,835, and ordinary rounding does not alter the resulting $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted estimated federal income tax and used the wrong $12,500 Louisiana deduction, reducing taxable income to $24,897. The applicable calculation subtracts only the $12,835 standard deduction from $40,000 AGI, yielding $814.95 at 3%."
+us,scenario_051,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first used the wrong $40,180 AGI and $12,500 deduction to compute $830.40, then submitted $1,080 despite claiming further deductions or credits would reduce liability. The correct base is $27,165 and the stated 3% rate produces $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete 4.25% rate and an unsupported $4,500 exemption to an incorrect $34,791 AGI. The 2026 calculation applies 3% to $40,000 less the $12,835 standard deduction."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an obsolete progressive schedule, deducted estimated federal tax, and substituted a $4,500 deduction from an incorrect $34,791 AGI. The applicable flat 3% rate is applied to $27,165 after the $12,835 standard deduction."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's explanation says it applied progressive Louisiana rates, but the 2026 computation uses a flat 3% rate. Taxable income is $27,165 after the $12,835 deduction, yielding $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an incorrect $34,791 AGI, a $4,500 exemption, and the obsolete 1.85%/3.5% brackets. The correct calculation uses $40,000 AGI, a $12,835 standard deduction, and a flat 3% rate."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model improperly deducted estimated federal income tax and $4,500, then applied an obsolete progressive rate schedule. Louisiana instead taxes $27,165 at a flat 3%, producing $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model started from an incorrect $34,791 AGI, allowed only a $4,500 deduction, and applied obsolete progressive brackets. The required base is $40,000 minus $12,835, taxed at 3%."
+us,scenario_051,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model added $180 to AGI, used only a $4,500 deduction, and applied the obsolete 1.85%/3.5% schedule. The 2026 calculation uses $40,000 AGI, the $12,835 deduction, and a flat 3% rate."
+us,scenario_051,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model added the $180 capital-gains input to AGI and used a $12,500 deduction. PolicyEngine's calculation uses $40,000 AGI and the $12,835 standard deduction, reducing the taxable base by $515 relative to the model."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions and exemptions eliminate the entire Louisiana liability. The $12,835 standard deduction leaves $27,165 taxable, so the 3% tax is $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model supplied only an unsupported estimate of $781 and did not apply the traced computation. Subtracting the $12,835 standard deduction from $40,000 leaves $27,165, whose 3% tax is $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of AGI and a $12,500 standard deduction rather than $40,000 and $12,835. Those errors produce $27,680 instead of the correct $27,165 taxable income."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model added the $180 input to AGI and understated the standard deduction by $335. The correct taxable income is $40,000 minus $12,835, or $27,165."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $40,180 as income and a $12,500 deduction. The calculation instead uses $40,000 AGI and a $12,835 deduction, leading to $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used a $27,680 taxable base derived from the wrong $40,180 AGI and $12,500 deduction. The correct taxable base is $27,165, taxed at 3%."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave an unsupported $680 estimate without identifying the applicable deduction or rate calculation. The traced computation is $40,000 minus $12,835, followed by 3% tax, yielding $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly used the federal standard deduction to determine Louisiana taxable income and then applied obsolete 1.85% and 3.5% brackets. Louisiana uses its $12,835 standard deduction and a flat 3% rate for this calculation."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model added $180 to AGI and used a $12,500 deduction instead of $12,835, then rounded to whole dollars. The exact computation is 3% of $27,165, or $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model started from an incorrect $34,791 AGI, deducted federal tax plus two separate $4,500 allowances, and applied obsolete progressive brackets. The correct computation uses only the $12,835 standard deduction against $40,000 AGI and applies 3%."
+us,scenario_051,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of AGI and a $12,500 deduction, producing the wrong taxable base, and then rounded cents away. The correct inputs produce $27,165 taxable income and exactly $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model did not return a value or explanation for the requested output. This is a missing-output contract failure rather than a substantive tax calculation.
+us,scenario_051,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model added the $180 input to AGI and used a $12,500 standard deduction. The correct Louisiana base is $40,000 less $12,835, yielding $814.95 at 3%."
+us,scenario_051,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model contradicted itself about whether Louisiana taxes income, used an obsolete $4,500 deduction, computed $1,065, and then submitted an unexplained $1,208. The applicable calculation uses the $12,835 deduction and produces $814.95."
+us,scenario_051,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used $40,180 of AGI and a $12,500 standard deduction. PolicyEngine uses $40,000 AGI and a $12,835 deduction, so taxable income is $27,165 rather than $27,680."
+us,scenario_051,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model substituted an estimated federal standard deduction, added a $4,500 personal exemption, and applied invented progressive rates instead of Louisiana's $12,835 deduction and flat 3% rate. Its submitted $573 also does not follow from its own stated $482.52 calculation."
+us,scenario_051,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model's $1,324.80 implies an incorrect taxable base or tax rate and provides neither the applicable deduction nor supporting arithmetic. The correct calculation taxes $27,165 at 3%, yielding $814.95."
 us,scenario_051,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's own calculation reached about $107,277 but it submitted $117,285 without a supporting computation. It also used estimated brackets and a $32,600 standard deduction instead of the 2026 parameters that produce $103,865.78 of regular tax."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $28,700 standard deduction instead of $32,200 and never produced a coherent bracket calculation for its submitted $103,241. The correct taxable income is $472,578.56, followed by $103,865.78 of regular tax, $127.20 of qualified-dividend tax, and $218.42 of NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model added $2,250 of Additional Medicare Tax to this income-tax output even though the benchmark assigns that employee-side tax to payroll_tax. It then replaced its approximately $107,040 calculation with an unsupported $110,997 adjustment based on estimated parameters."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model calculated AGI as about $505,748 because it failed to subtract the $926 pre-tax 401(k) contribution in its displayed arithmetic. That inflated taxable income above the correct $472,578.56, and it also omitted the $218.42 NIIT from the submitted total."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model stated taxable income of about $472,200 but miscomputed the tax on that income as $141,500. Applying the 2026 joint brackets yields $103,865.78 of regular tax, not a liability near $141,500."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model incorrectly included $2,250 of Additional Medicare Tax in federal_income_tax_before_refundable_credits; that amount belongs in the benchmark's payroll_tax output. It also used a $30,000 estimated standard deduction instead of $32,200."
-us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,period_annualization,False,"The prompt states that all listed amounts are annual, but the model annualized the $926 401(k) and $43 IRA entries by multiplying them by 52. It then discarded its own roughly $94,754 calculation and submitted an unsupported $121,500."
-us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted the $16,208 employer-sponsored insurance premium from wages even though the reference wage adjustment is only the $926 pre-tax 401(k) contribution. This reduced AGI to $488,614 instead of $504,778.56; it also used $8,748 rather than $5,748 as the NIIT base."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented $45,620 of itemized mortgage-interest and SALT deductions even though no mortgage interest or deductible tax payment was listed. It also applied an assumed TCJA sunset rather than the operative 2026 brackets and $32,200 standard deduction."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model reduced income by essentially the full capital loss instead of limiting the deduction against ordinary income to $3,000, producing an erroneous AGI near $476,323. Its $139,198 tax does not follow from either that AGI or the correct $472,578.56 taxable income."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted $123,753 implies that the model applied the wrong 2026 joint brackets or tax regime. The correct components are $103,865.78 of regular tax, $127.20 on qualified dividends, and $218.42 of NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a post-TCJA-expiration regime with a $15,300 standard deduction and personal exemptions instead of the operative $32,200 joint standard deduction and 2026 brackets. It also improperly deducted the $16,208 employer premium from wages."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model's $113,421 total reflects incorrect 2026 bracket or deduction parameters. On $472,578.56 of taxable income, the specified components total $104,211.41."
-us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model used AGI of $488,614, which results from wrongly subtracting the $16,208 employer-sponsored insurance premium from the stated wages. The correct AGI is $504,778.56 and the correct NIIT-inclusive total is $104,211.41."
-us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model assumed TCJA expiration, using pre-TCJA rates, a $15,900 standard deduction, and personal exemptions. The operative 2026 rules instead provide a $32,200 joint standard deduction and brackets producing $103,865.78 of regular tax."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model drastically undercomputed the progressive tax on roughly $472,579 of joint taxable income. The 2026 brackets alone produce $103,865.78 of regular tax before adding qualified-dividend tax and NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the small retirement contributions and health premiums as eliminating tax on more than $500,000 of AGI. They do not: taxable income remains $472,578.56, and there are no nonrefundable credits that reduce the resulting liability to zero."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $472,622 of taxable income because it omitted the additional $43 of above-the-line deductions included in the $3,043.28 total. It also failed to add the $218.42 NIIT to the final income-tax output."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used $472,622 rather than $472,578.56 of taxable income, omitting the remaining $43.44 adjustment reflected in the trace. Its explanation also omitted the $218.42 NIIT required in this output."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the wrong NIIT base, reporting about $332 consistent with taxing all $8,748 of interest and dividends without the $3,000 net-capital-loss reduction. Net investment income is $5,748 and NIIT is $218.42."
-us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly identified the major income components but severely undercomputed tax on $472,578.56 of taxable income. The 2026 joint brackets yield $103,865.78 of regular tax, not $57,891."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a rough effective-rate estimate rather than applying the 2026 married-filing-jointly brackets. The bracket computation produces $103,865.78 of regular tax, with only $345.62 added for qualified dividends and NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied projected post-TCJA rules, including a $16,600 standard deduction and personal exemptions, instead of the operative $32,200 deduction and 2026 brackets. It also wrongly subtracted the $16,208 employer premium from wages and used $8,748 instead of $5,748 for NIIT."
-us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from gross wages, reducing AGI by that amount. It also calculated NIIT on $8,748 instead of reducing net investment income by the $3,000 capital-loss deduction to $5,748."
-us,scenario_052,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used $472,622 rather than $472,578.56 of taxable income and overstated the ordinary bracket tax. The correct regular tax is $103,865.78; adding $127.20 and $218.42 yields $104,211.41."
-us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured answer was missing."
-us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $4,517.80 overtime deduction from the work schedule even though gross wages already include overtime and the trace contains no such deduction. That reduced taxable income from $472,578.56 to $468,104.20 and understated the tax."
-us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model gave contradictory standard deductions of $16,100 and $30,800 and never consistently derived taxable income. The required deduction is $32,200, producing taxable income of $472,578.56 and total tax of $104,211.41."
-us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the full $29,922 net capital loss in computing AGI instead of limiting the current-year ordinary-income deduction to $3,000. It also produced a submitted $165,473.58 that contradicts its own stated $85,184.10 regular tax and zero AMT."
-us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied an erroneous shortcut of $76,256 plus 35% of income above $256,800, rather than the actual 2026 married-filing-jointly bracket schedule. It also omitted the $926 pre-tax 401(k) reduction and mishandled the qualified-dividend rate adjustment."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's bracket calculation was already far above the engine's $103,865.78 main tax, and it then submitted $117,285 despite its own derived total of $107,277. It failed to apply the exact 2026 MFJ brackets and violated the answer-value consistency requirement."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model initially deducted the entire $29,922 capital loss and used a $28,700 standard deduction, then switched to the $3,000 loss limit without consistently rebuilding the calculation. The correct taxable income is $472,578.56, followed by $103,865.78 of main tax, $127.20 of qualified-dividend tax, and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model added $2,250 of Additional Medicare Tax to this income-tax output even though the benchmark assigns it to payroll_tax. It also used estimated brackets and deductions rather than the exact 2026 parameters, then submitted $110,997 instead of its own $107,040 calculation."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model stated an AGI of about $505,748 even though subtracting the $926.16 pre-tax 401(k) contribution yields wages of $499,073.84, and it omitted the remaining above-the-line deduction reflected in the $3,043.28 total. It also stopped at regular tax and omitted the $218.42 NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied the 2026 MFJ rate schedule incorrectly: taxable income near $472,579 produces $103,865.78 of main tax, not $141,500. It also failed to add the separately computed $127.20 qualified-dividend tax and $218.42 NIIT to a correctly calculated main tax."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model included the $2,250 Additional Medicare Tax in federal_income_tax_before_refundable_credits, although this benchmark includes that amount in payroll_tax. It also used a $30,000 estimated standard deduction instead of $32,200, inflating taxable income."
+us,scenario_052,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,period_annualization,False,"The model annualized the explicitly annual $926 traditional 401(k) and $43 IRA inputs by multiplying them by 52, contrary to the prompt's full-year convention. It then abandoned its computed $94,754 result and submitted an unsupported $121,500."
+us,scenario_052,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted the $16,208 employer-sponsored insurance premium from the separately stated gross wages even though the engine only reduces wages by the $926.16 pre-tax 401(k) contribution. It also used a $30,800 rather than $32,200 standard deduction and calculated NIIT on $8,748 instead of net investment income of $5,748 after the $3,000 capital-loss deduction."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model invented $45,620 of mortgage-interest and state/local-tax deductions despite the prompt listing neither mortgage interest paid nor deductible taxes. It also applied a projected TCJA-sunset rate regime instead of the applicable 2026 brackets."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated AGI of approximately $476,323 treats far more than the permitted $3,000 net capital loss as currently deductible. The engine limits the $29,922 capital loss against ordinary income and derives taxable income of $472,578.56 before applying the 2026 rates."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted $123,753 implies an incorrect 2026 rate-schedule computation. Taxable income of $472,578.56 yields $103,865.78 of main tax, with only $127.20 of qualified-dividend tax and $218.42 of NIIT added."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration, using a $15,300 standard deduction, personal exemptions, and the old rate structure. It also improperly subtracted the $16,208 employer-sponsored premium from gross wages; the applicable calculation uses the $32,200 joint standard deduction and current 2026 brackets."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $113,421 does not result from the applicable 2026 MFJ brackets. The correct bracket computation on $472,578.56 is $103,865.78, followed by $127.20 of qualified-dividend tax and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's $488,614 AGI improperly subtracts the $16,208 employer-sponsored premium from the stated gross wages. The engine uses $499,073.84 of employment income after only the pre-tax 401(k) contribution and derives taxable income of $472,578.56."
+us,scenario_052,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with a $15,900 standard deduction, personal exemptions, and 10/15/25/28/33/35 percent brackets. The applicable 2026 calculation uses a $32,200 joint standard deduction and the retained 10/12/22/24/32/35/37 percent schedule."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $50,504 answer severely underapplies the progressive MFJ rate schedule to $472,578.56 of taxable income. The brackets alone produce $103,865.78 before adding qualified-dividend tax and NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the small retirement contributions and health premiums as eliminating tax on roughly $500,000 of wages. Those inputs leave $472,578.56 of taxable income and no nonrefundable credits, so substantial federal income tax remains."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model derived taxable income of $472,622 instead of $472,578.56 because it omitted the remaining $43.28 of the engine's $3,043.28 above-the-line deductions. It also reported only a rounded regular-tax result rather than adding $127.20 of qualified-dividend tax and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used $472,622 rather than $472,578.56 of taxable income, omitting $43.28 from the engine's above-the-line deductions. Its explanation also omitted the $218.42 NIIT required in this output."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used the wrong taxable-income base and calculated NIIT as about $332 by failing to net the $3,000 capital-loss deduction against investment income. Net investment income is $5,748, so NIIT is $218.42."
+us,scenario_052,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model drastically undercomputed the 2026 MFJ bracket tax on taxable income of $472,578.56. The main tax is $103,865.78, not $57,891, before the qualified-dividend tax and NIIT additions."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a rough effective-rate estimate instead of the 2026 MFJ bracket calculation. The exact schedule produces $103,865.78 of main tax, and the two additional income-tax components total only $345.62."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and applied pre-TCJA rates, a $16,600 standard deduction, and personal exemptions. It also improperly excluded the $16,208 employer-sponsored premium from stated gross wages and used $8,748 rather than $5,748 as the NIIT base."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $30,750 standard deduction instead of the applicable $32,200 and omitted $43.28 from the engine's above-the-line deductions. Those errors raised taxable income from $472,578.56 to $474,072 and overstated the tax."
+us,scenario_052,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $16,208 employer-sponsored premium from gross wages and separately deducted the $43 IRA amount. It also used a $30,750 standard deduction and an $8,748 NIIT base instead of the $32,200 deduction and $5,748 net-investment-income base."
+us,scenario_052,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used $472,622 rather than $472,578.56 of taxable income and overstated the ordinary bracket tax as about $104,035. The applicable main-tax calculation is $103,865.78, to which $127.20 and $218.42 are added."
+us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_052,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $4,517.80 overtime deduction from the listed work schedule even though the wage amount is already an annual total including overtime and the engine trace contains no such deduction. That unsupported deduction reduced taxable income below the correct $472,578.56, and the model also omitted NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model alternated between $16,100 and $30,800 standard deductions and misstated AGI as an amount after the standard deduction. It never consistently derived the engine's $472,578.56 taxable income or added the $218.42 NIIT to the correctly computed tax."
+us,scenario_052,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model treated the $848 qualified dividend as taxed at 0 percent even though this household's taxable income places it in the 15 percent qualified-dividend band. Its $103,866 answer is essentially the main tax alone and omits both $127.20 of dividend tax and $218.42 of NIIT."
+us,scenario_052,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the full $29,922 capital loss in computing AGI instead of limiting the current ordinary-income deduction to $3,000. It then submitted $165,473.58 despite its own regular-tax calculation of $85,184.10 and its conclusion that no AMT was due."
+us,scenario_052,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model applied 35 percent to a fabricated $218,948 bracket segment and produced a regular-tax figure inconsistent with the progressive 2026 MFJ schedule. It also failed to subtract the pre-tax 401(k) contribution and incorrectly described the qualified-dividend adjustment as adding a 15 percent differential after taxing all income as ordinary."
 us,scenario_052,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_052,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified benchmark rule making a 59-year-old Medicare eligible. Medicare's standard age threshold is 65, and the prompt supplies no disability, end-stage renal disease, or other alternative eligibility basis, so the head is not eligible."
-us,scenario_052,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $16,208 employer-sponsored insurance premium as a reduction to FICA wages even though the input does not establish an employee pre-tax cafeteria-plan deduction. It then submitted $12,820.40, which does not equal either its erroneous $20,558.11 calculation or the correctly calculated $20,939."
-us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used an obsolete $168,600 Social Security wage base and the $200,000 employer-withholding threshold instead of the $250,000 joint-return threshold for Additional Medicare Tax. It then submitted $38,335.50 despite its own components totaling $20,403.20."
-us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used $176,100 rather than the 2026 Social Security wage base of $184,500. This understated Social Security tax by $520.80 while its Medicare components were calculated correctly."
-us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied the $200,000 per-employee withholding threshold to determine Additional Medicare Tax instead of the $250,000 threshold for a joint return. That produced $2,700 rather than $2,250 of Additional Medicare Tax, overstating payroll tax by $450."
-us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly derived all three components and their $20,939 sum, then replaced that result with an unsupported rounded value of $21,118. Payroll tax must equal the stated component sum without this extra $179."
-us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model carried the 2025 Social Security wage base of $176,100 into 2026 instead of using the 2026 base of $184,500. Its regular and Additional Medicare calculations were correct, but Social Security tax was understated by $520.80."
-us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model alternated between an obsolete Social Security wage base and the wrong $200,000 Additional Medicare threshold, producing an intermediate $20,868.20. It then submitted $16,856.10, which is unsupported by its own component calculations and omits $4,082.90 of the correct payroll tax."
-us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from FICA wages without an input establishing a pre-tax employee salary reduction. It also used an incorrect $181,000 Social Security wage base instead of $184,500."
-us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a projected $171,300 Social Security wage base rather than the 2026 value of $184,500. This reduced Social Security tax from $11,439 to $10,620.60 while the two Medicare components were correct."
-us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model calculated Additional Medicare Tax as $18,958.50 instead of applying the 0.9% rate to the $250,000 of wages above the joint threshold, which yields $2,250. It also understated capped Social Security tax as $9,932.50 instead of $11,439."
-us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted total implies that the model used the wrong Social Security wage-base amount while including regular and Additional Medicare tax. With $7,250 and $2,250 of Medicare taxes, the correct $11,439 Social Security component produces $20,939, not $20,279."
-us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly deducted the $16,208 employer-sponsored insurance premium from Medicare and Additional Medicare wages even though no pre-tax employee salary reduction was specified. It also used an obsolete estimated $172,200 Social Security wage base rather than $184,500."
-us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted total is $586 below the result of applying the stated components to the governing bases. Regular Medicare tax is $7,250, Additional Medicare Tax is $2,250, and capped Social Security tax is $11,439, which sum to $20,939."
-us,scenario_052,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages to $483,792 by treating the employer-sponsored insurance premium as a pre-tax employee deduction. It also understated Social Security tax as $11,271.60 instead of applying 6.2% to the $184,500 wage base for $11,439."
-us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the desired traditional 401(k) contribution; employee elective deferrals remain subject to Social Security and Medicare taxes. It also used a $183,000 Social Security wage base rather than $184,500."
-us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model incorrectly stated that $500,000 of wages were below the $250,000 joint threshold and therefore omitted the $2,250 Additional Medicare Tax. Its $38,897 submission also exceeds the correct employee-side Social Security and Medicare components and is consistent with including unsupported payroll-tax amounts."
-us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a payroll-tax-applicability requirement and treated explicitly listed wages as exempt from FICA. The $500,000 of wages generate $11,439 of Social Security tax, $7,250 of Medicare tax, and $2,250 of Additional Medicare Tax."
-us,scenario_052,payroll_tax,grok-4.3,llm_error,other,False,"The model's stated components cannot total approximately $13,500: regular Medicare tax alone is $7,250 and capped Social Security tax is $11,439 before adding $2,250 of Additional Medicare Tax. It therefore made a fundamental summation error after identifying all three required taxes."
-us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly reduced Medicare and Additional Medicare wages by the $16,208 employer-sponsored insurance premium without a specified pre-tax employee salary reduction. Applying both Medicare taxes to the full $500,000 of wages adds the omitted $381 and produces $20,939."
-us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from Medicare wages and used a projected $183,144 Social Security wage base. The applicable bases are the full $500,000 for Medicare and $184,500 for Social Security."
-us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, so the required numeric result was missing."
-us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly identified components of $11,439, $7,250, and $2,250, although it also misstated their sum as $30,939 rather than $20,939. It then submitted $11,573 after an unsupported adjustment for a nonexistent spouse wage portion."
-us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model repeatedly calculated $20,418.20 from its chosen components but submitted $32,846 without any supporting computation. Separately, its $20,418.20 intermediate result used the obsolete $176,100 Social Security wage base instead of $184,500."
-us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model added the $2,250 Additional Medicare Tax twice: its stated $20,418.20 subtotal already included that component, yet it added another $2,250 to reach $22,668.20. It also used the obsolete $176,100 Social Security wage base, understating that component by $520.80."
+us,scenario_052,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly computed the full-wage components as $11,439, $7,250, and $2,250 but then submitted $12,820.40, which does not equal either of its stated totals. It discarded its own payroll-tax derivation when producing the numeric output."
+us,scenario_052,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model used an obsolete $168,600 Social Security wage base and the $200,000 employer-withholding threshold instead of the $250,000 joint-return Additional Medicare Tax threshold. It then submitted $38,335.50 even though its stated components sum to $20,403.20."
+us,scenario_052,payroll_tax,claude-opus-4.7,llm_error,thresholds_rates,False,"The model capped Social Security wages at $176,100 rather than the 2026 wage base of $184,500. This understated employee Social Security tax by $520.80 while its Medicare calculations were correct."
+us,scenario_052,payroll_tax,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied the $200,000 employer withholding trigger as though it were the household's Additional Medicare Tax threshold. A married couple filing jointly uses $250,000, so the additional tax is $2,250 rather than $2,700."
+us,scenario_052,payroll_tax,claude-opus-5,llm_error,other,False,"The model's stated components correctly total $20,939, but it submitted $21,118 and called that a rounded result. $21,118 is not a rounding of $20,939 and has no support in its calculation."
+us,scenario_052,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model reused the 2025 Social Security wage base of $176,100 for 2026 after cycling through unsupported projections. The required 2026 base is $184,500, producing $11,439 of Social Security tax rather than $10,918.20."
+us,scenario_052,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model alternated between the wrong $200,000 Additional Medicare Tax threshold and an obsolete Social Security wage base, then submitted $16,856.10 despite explicitly calculating $20,868.20. The submitted amount is unsupported by its stated components."
+us,scenario_052,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from the benchmark's $500,000 payroll-tax wage base and also used a $181,000 Social Security cap. PolicyEngine applies regular Medicare and Additional Medicare Tax to the full wages and caps Social Security wages at $184,500."
+us,scenario_052,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a projected $171,300 Social Security wage cap instead of the 2026 $184,500 cap. This reduced Social Security tax to $10,620.60 instead of $11,439."
+us,scenario_052,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model calculated $18,958.50 of Additional Medicare Tax on only $250,000 of excess wages, effectively applying a 7.5834% rate instead of 0.9%. The correct additional tax is $2,250, and its $9,932.50 Social Security component also fails to use the $184,500 wage base."
+us,scenario_052,payroll_tax,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model included the three proper categories but understated their total by $660. With regular Medicare and Additional Medicare Tax fixed at $7,250 and $2,250, its answer implies only $10,779 of Social Security tax rather than $11,439 under the 2026 wage base."
+us,scenario_052,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages by the $16,208 employer-sponsored insurance premium, lowering both Medicare components, and used a $172,200 Social Security cap. The computation uses full $500,000 wages for Medicare and the $184,500 Social Security wage base."
+us,scenario_052,payroll_tax,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model named all three components and the correct joint threshold but understated their combined value by $586. Its total therefore fails to apply the $184,500 Social Security wage base and the stated Medicare rules to the full $500,000 of wages."
+us,scenario_052,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model improperly reduced Medicare wages to $483,792 by subtracting the employer-sponsored insurance premium, producing $7,014.98 and $2,104.13 instead of $7,250 and $2,250. It also used a Social Security component of $11,271.60 rather than the $11,439 amount generated from the 2026 wage base."
+us,scenario_052,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly treated the $926 traditional 401(k) contribution as reducing Social Security and Medicare wages; elective 401(k) deferrals do not reduce FICA wages. It also used a $183,000 Social Security wage base instead of $184,500."
+us,scenario_052,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model wrongly stated that $500,000 of joint wages are below the $250,000 Additional Medicare Tax threshold and omitted the resulting $2,250 tax. Its $38,897 total also exceeds the capped employee Social Security and Medicare amounts, consistent with failing to enforce the Social Security wage cap."
+us,scenario_052,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model invented a payroll-tax-applicability condition and treated the listed $500,000 of wages as exempt from employee FICA. Listed wages are sufficient to generate Social Security, regular Medicare, and Additional Medicare Tax totaling $20,939."
+us,scenario_052,payroll_tax,grok-4.3,llm_error,other,False,"The model named Social Security, regular Medicare, and Additional Medicare Tax but submitted $13,500, which is below the sum of those components even using its own approximate $168,000 cap. Applying the 2026 $184,500 cap and the stated Medicare rules yields $20,939."
+us,scenario_052,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from Medicare wages. Regular and Additional Medicare Tax apply here to the full $500,000, producing $7,250 and $2,250 rather than $7,015 and $2,104."
+us,scenario_052,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the $926 traditional 401(k) deferral. That deferral does not reduce Social Security or Medicare wages, so the Medicare components must be computed on the full $500,000."
+us,scenario_052,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted the $16,208 employer-sponsored insurance premium from Medicare wages and used a projected $183,144 Social Security cap. The trace uses $500,000 for both Medicare components and the $184,500 Social Security wage base."
+us,scenario_052,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required structured-output contract."
+us,scenario_052,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly stated components of $11,439, $7,250, and $2,250, which sum to $20,939, but then misstated their sum as $30,939 and submitted $11,573. The invented adjustment for a nonexistent spouse wage portion has no role in household employee payroll tax."
+us,scenario_052,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model used the obsolete $176,100 Social Security wage base and repeatedly calculated a $20,418.20 total, then submitted $32,846 without any supporting computation. The output is inconsistent with both its reasoning and the required 2026 wage base."
+us,scenario_052,payroll_tax,qwen3.8-max,llm_error,other,False,"The model first included the $2,250 Additional Medicare Tax in its stated $20,418.20 component sum and then added that same $2,250 a second time. It also used the obsolete $176,100 Social Security wage base rather than $184,500."
 us,scenario_052,spouse_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model invented a simplified rule that makes a 55-year-old Medicare eligible. The spouse is below the age-65 threshold and has no listed disability or other qualifying Medicare pathway, yielding spouse_medicare_eligible = 0."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model improperly deducted $3,389 of employer-sponsored premiums from wages even though the stated gross wages are already the annual tax input, and it also introduced an auto-loan-interest deduction that the trace does not apply. Its submitted $6,224 further contradicts its own final calculation of $5,335.60."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model deducted the ESI premium from wages, used a $14,600 standard deduction instead of $16,100, and wrote an invalid bracket computation that applies 22% across income belonging in lower brackets. Its submitted $5,744 does not follow from its stated $12,761.60 calculation."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $16,600 standard deduction instead of $16,100 and then abandoned its own bracket calculation of about $5,796 for an unsupported $6,196 output. The correct taxable income is $50,868.68 and the bracket calculation yields $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly computed the correct bracket tax of about $5,903 from the correct deduction and thresholds, then submitted $6,027 without any calculation supporting that adjustment. This is a final-answer arithmetic inconsistency."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model explicitly derived about $5,903 using the correct $16,100 deduction and 2026 bracket structure, then replaced it with $6,379 without a supporting computation. The submitted value contradicts its own derivation."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model cycled through estimated standard deductions and ultimately used $15,700 instead of $16,100, while also using lower estimated bracket cutoffs of $11,925 and $48,475. Those substitutions put too much taxable income into the 22% bracket and produce $6,193 instead of applying the specified 2026 parameters."
-us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although it used the $16,100 standard deduction, the model inflated projected 2025 bracket thresholds rather than applying the 2026 thresholds of $12,400 and $50,400. The correct brackets tax only $468.68 at 22%, yielding $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the stated gross wages by $3,389 of ESI premiums and used a $15,400 standard deduction rather than $16,100. AGI remains $66,968.68, so taxable income is $50,868.68 rather than $48,180."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and applied 10%, 15%, and 25% brackets plus an obsolete-law deduction-and-exemption structure. The applicable 2026 calculation uses the $16,100 standard deduction and 10%, 12%, and 22% brackets."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $3,939 is incompatible with the stated $66,969 income, $15,300 deduction, and ordinary 2026 single brackets. Using the actual $16,100 deduction and 2026 thresholds produces $50,868.68 of taxable income and $5,903.11 of tax."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration and used the pre-TCJA deduction, personal-exemption, and rate structure. It also deducted ESI premiums from the stated wages; the applicable calculation keeps $66,968.68 as AGI and applies the $16,100 deduction and 2026 10%/12%/22% brackets."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an expired-law scenario with an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% marginal brackets. The applicable 2026 parameters are a $16,100 standard deduction and 10%, 12%, and 22% brackets."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly subtracted $3,389 of employer-sponsored premiums from the stated gross wages, reducing taxable income to $47,480. The wage input remains $66,968.68 for AGI, so the $16,100 standard deduction leaves $50,868.68 taxable."
-us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model treated the separately listed employer-sponsored premium as an additional exclusion from the stated annual gross wages. That shortcut produces $47,480 of taxable income and $5,449.60, whereas the correct taxable income is $50,868.68."
-us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction and obsolete bracket thresholds of $11,600 and $47,150. The 2026 calculation instead uses a $16,100 deduction with thresholds of $12,400 and $50,400."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invoked a basic CTC or other nonrefundable credits despite the household having no child, dependent, or other credit-qualifying fact. No nonrefundable credit applies, and the regular tax on $50,868.68 is $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used unspecified approximations for the standard deduction and brackets that understated the regular tax. Applying the exact $16,100 deduction and 2026 single thresholds to $66,968.68 of AGI yields $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $16,000 deduction and then misapplied the 2026 brackets, since its stated $50,969 taxable income would not produce $5,793 under those brackets. The exact $16,100 deduction leaves $50,868.68 taxable and produces $5,903.11."
-us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly identified taxable income near $50,869 but miscalculated the 10%, 12%, and 22% bracket tax. Applying those brackets to $50,868.68 yields $5,903.11, not $5,985."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an estimated $15,000 standard deduction and an unsupported bracket approximation. The actual $16,100 deduction and exact 2026 thresholds yield $50,868.68 of taxable income and $5,903.11 of tax."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed TCJA sunset parameters, including an $8,300 deduction, a $5,300 personal exemption, and 15% and 25% brackets. The applicable 2026 calculation uses the $16,100 standard deduction and 10%, 12%, and 22% brackets, with no personal exemption added."
-us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied pre-TCJA law, used only an $8,372 standard deduction, and taxed the resulting $58,597 under 10%, 15%, and 25% brackets. The applicable $16,100 deduction leaves $50,868.68 taxable under the 10%, 12%, and 22% structure."
-us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model assumed TCJA expiration and substituted an $8,300 deduction, a $5,300 personal exemption, and 15% and 25% brackets. It also deducted ESI premiums from stated wages; the correct calculation starts from $66,968.68 of AGI and applies the $16,100 deduction and current 2026 brackets."
-us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model mislabeled a $13,850 standard deduction as an AGI adjustment and used that outdated amount instead of $16,100. It then calculated $6,397.98 from its own assumptions but submitted $4,272, so the final answer also contradicts its stated arithmetic."
-us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's written bracket calculation totals $6,127.18, yet it submitted $8,252 without any supporting adjustment or credit computation. It also used estimated deduction and bracket values instead of the exact $16,100 deduction and $12,400/$50,400 thresholds."
-us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model reduced regular tax by a child or dependent credit even though the household contains no child or dependent and has no other nonrefundable-credit facts. No such credit applies, so the full regular tax of $5,903.11 remains before refundable credits."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model deducted $3,389 of employer-sponsored premiums and $950 of auto-loan interest even though gross wages are the taxable wage input and no deductible treatment was specified. More decisively, it submitted $6,224 despite its own final calculation producing $5,335.60, so the reported value does not follow from its reasoning."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of $16,100 and deducted the separately listed employer premium from gross wages. Its displayed bracket calculation is also internally invalid and yields $12,761.60 rather than the submitted $5,744."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted an estimated $16,600 standard deduction for the applicable $16,100 amount. Its own bracket arithmetic produced about $5,796, but it then submitted $6,196 without any computation supporting that increase."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived approximately $5,903 from $50,869 of taxable income and the applicable 10%, 12%, and 22% bracket segments. It then discarded that result and submitted $6,027, an unsupported final-value substitution."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly computed approximately $5,903 using the $16,100 deduction and the applicable brackets. It nevertheless submitted $6,379, which is disconnected from every figure in its stated derivation."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted projected standard deductions and estimated bracket thresholds instead of applying the 2026 $16,100 deduction and applicable brackets. That parameter drift left taxable income too high and produced $6,193 instead of taxing $50,868.68 to $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although it used the correct approximate taxable income, the model applied projected thresholds derived by inflating older brackets rather than the applicable 2026 thresholds. This misplaced income between the 12% and 22% bands and overstated the tax."
+us,scenario_053,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the separately listed $3,389 employer premium from gross wages and used a $15,400 standard deduction rather than $16,100. The calculation must start from $66,968.68 of AGI and $50,868.68 of taxable income."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, restored a personal exemption, and applied 15% and 25% brackets. The applicable 2026 computation uses the $16,100 standard deduction and the 10%, 12%, and 22% bracket schedule."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $3,939 does not result from the stated $66,969 income, estimated $15,300 deduction, and ordinary 2026 single brackets. Applying the actual $16,100 deduction to $66,968.68 and then the applicable brackets yields $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed TCJA expiration and used the pre-TCJA rate structure, while also subtracting the employer premium from gross wages. The applicable computation retains the $16,100 standard deduction and current 10%, 12%, and 22% brackets."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model incorrectly restored the personal exemption and substituted post-sunset 15% and 25% rates for the applicable 12% and 22% bands. It also reduced gross wages by the separately stated employer premium without a specified pre-tax wage adjustment.
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the wage input by $3,389 for employer-sponsored premiums. Using the full $66,968.68 employment income and the $16,100 standard deduction gives $50,868.68 of taxable income, not $47,480."
+us,scenario_053,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer reflects subtracting the $3,389 employer premium from the supplied gross-wage amount before applying the standard deduction. That premium is not a separate adjustment to the wage input, so taxable income is $50,868.68 and tax is $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used the obsolete $14,600 standard deduction and older $11,600 and $47,150 bracket thresholds. The 2026 calculation uses a $16,100 deduction and taxes $50,868.68 under the applicable thresholds."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invoked a basic CTC or other nonrefundable-credit reduction despite the household having no child, dependent, or other stated credit eligibility. With no nonrefundable credits, the full regular tax of $5,903.11 remains."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used unspecified approximations for the standard deduction and brackets that understated the regular tax. The exact 2026 inputs are a $16,100 deduction, $50,868.68 of taxable income, and $5,903.11 of tax."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an estimated $16,000 standard deduction and then misapplied the bracket calculation: its $5,793 result is below the tax produced even with its slightly higher taxable income. The applicable parameters yield $50,868.68 of taxable income and $5,903.11 of tax."
+us,scenario_053,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the correct rounded taxable income but calculated the bracket tax incorrectly. Applying the 10%, 12%, and 22% segments to $50,868.68 produces $5,903.11, not $5,985."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an approximate $15,000 standard deduction instead of $16,100 and did not show a valid bracket calculation for its $5,850 result. The exact taxable income is $50,868.68 and its regular tax is $5,903.11."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly treated 2026 as governed by post-TCJA-sunset parameters, restoring a personal exemption and applying 15% and 25% brackets. The applicable schedule uses the $16,100 standard deduction with 10%, 12%, and 22% marginal rates at this income."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and used an $8,522 standard deduction plus pre-TCJA-style 15% and 25% rates. The applicable 2026 calculation instead deducts $16,100 and taxes $50,868.68 under the current bracket structure."
+us,scenario_053,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a pre-TCJA $8,372 standard deduction, omitted the applicable $16,100 deduction, and applied 15% and 25% rates. Those obsolete parameters overstated taxable income and tax substantially."
+us,scenario_053,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model assumed TCJA expiration, restored a personal exemption, and applied pre-TCJA 15% and 25% brackets. It also subtracted the separately listed employer premium from gross wages; the applicable calculation starts with $66,968.68 and uses the $16,100 standard deduction."
+us,scenario_053,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model used a $13,850 deduction instead of $16,100 and explicitly calculated tax of $6,397.98 from that premise. It then submitted $4,272, a number that contradicts its own arithmetic and has no stated computation."
+us,scenario_053,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's stated approximate computation produced $6,127.18 and then $6,127, but it submitted $8,252. The final value is an unsupported substitution and does not follow from its taxable-income or bracket calculations."
+us,scenario_053,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model reduced regular tax for a child or dependent credit even though this single-person household has no child or dependent. No nonrefundable credit applies, so taxing $50,868.68 under the 2026 brackets leaves $5,903.11."
 us,scenario_053,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a refundable child or other-dependent credit despite the household containing no qualifying child or dependent, and it also treated this high-income childless filer as receiving refundable EITC. Neither eligibility pathway applies, so the claimed $1,966 credit has no qualifying component."
 us,scenario_053,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model treated the $3,389 employer-sponsored insurance premium as a pre-tax payroll deduction and reduced FICA wages to $63,580. The prompt never identifies that premium as pre-tax, so Social Security and Medicare apply to the full $66,969 of wages."
 us,scenario_053,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $3,389 employer-sponsored insurance premium from gross wages before applying FICA. Because pre-tax treatment was not listed, the applicable Social Security and Medicare wage base is $66,969, not $63,580."
@@ -3306,71 +3475,76 @@ us,scenario_053,payroll_tax,gpt-5.4-mini,llm_error,thresholds_rates,False,"The m
 us,scenario_053,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model stated the correct 6.2% Social Security and 1.45% Medicare rates and correctly excluded Additional Medicare Tax, but its arithmetic does not equal those rates applied to $66,969. A combined 7.65% calculation is approximately $5,123.1, not $5,329."
 us,scenario_053,payroll_tax,kimi-k2.6,llm_error,payroll_tax_base,False,"The model assumed the $3,389 employer-sponsored insurance premium was a pre-tax FICA exclusion and reduced taxable payroll wages to $63,580. No pre-tax treatment is supplied, so both Social Security and Medicare apply to the full $66,969."
 us,scenario_053,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model abandoned its correctly identified $50,869 taxable-income base and instead applied 5.695% to an invented $53,000 base. It failed to apply Idaho's 2026 zero-rate bracket and operative rate structure, which yield $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used obsolete graduated brackets reaching 7.15%, an incorrect $7,200 deduction, and an unsupported $59 exemption credit. Idaho's 2026 calculation uses $50,868.68 of taxable income and the current rate structure, producing $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model guessed among 4.7%, 5.3%, and 5.695% instead of applying Idaho's actual 2026 rate structure. It also used a $15,750 deduction rather than $16,100, so it did not reach the $50,868.68 taxable-income base taxed to $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model identified taxable income near $50,869 but used the obsolete 5.695% rate and then reported an amount inconsistent with its own displayed arithmetic. Applying the 2026 Idaho zero-rate bracket and operative rate to $50,868.68 yields $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated the Idaho grocery credit as a nonrefundable offset in this before-refundable-credits output. It also failed to compute the 2026 zero-rate bracket precisely; the tax on $50,868.68 before refundable credits is $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the listed employer-sponsored insurance premium from gross wages even though the trace AGI is $66,968.68, then used an estimated $15,000 deduction. It also applied 5.695% to every taxable dollar rather than using Idaho's zero-rate bracket, while the correct $50,868.68 base produces $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $3,706 contradicts the model's own calculation of approximately $2,960 and has no supporting computation. The required calculation uses $50,868.68 of taxable income and Idaho's 2026 rate structure to produce $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable-income base of $48,180 instead of subtracting the $16,100 deduction from $66,968.68 to obtain $50,868.68. It then applied 5.3% to the entire base without the Idaho zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an unsupported $49,980 taxable-income base and applied the obsolete 5.695% rate to every dollar. Idaho's 2026 zero-rate bracket applied to the traced $50,868.68 base yields $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The estimate does not identify or apply Idaho's 2026 zero-rate bracket and operative rate. The traced deduction leaves $50,868.68 taxable, whose computed tax is $2,435.28 rather than $2,652."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied an obsolete 5.692% flat rate to an unsupported $49,852 base. It omitted Idaho's 2026 zero-rate bracket, which is essential to the $2,435.28 result on $50,868.68 of taxable income."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI to $63,580 for employer premiums and then used an unexplained $49,980 taxable base. It compounded that error by applying an obsolete 5.8% rate to every dollar instead of the 2026 Idaho rate structure."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $3,389 of employer-sponsored premiums from the traced AGI before taking the standard deduction, reducing taxable income to $47,480 rather than $50,868.68. It also applied 5.695% across the full base and omitted the zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer is consistent with multiplying roughly $47,470 by the obsolete 5.695% rate, not with Idaho's 2026 tax computation. The correct calculation applies the zero-rate bracket and operative rate to $50,868.68, yielding $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a deduction equal to $1,250 per $1,000 of federal taxable income and thereby eliminated all taxable income. Idaho instead allows the traced $16,100 deduction, leaving $50,868.68 taxable and $2,435.28 of tax."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model used an unspecified bracket estimate that understated the liability. Idaho's 2026 rate structure applied to $50,868.68 of taxable income produces $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model supplied a rough $1,195 estimate without applying Idaho's actual 2026 rate structure. The $16,100 deduction leaves $50,868.68 taxable, and the resulting tax is $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $15,750 standard deduction and a $4,489 zero bracket instead of the 2026 parameters. Those errors produced $51,219 of taxable income and the wrong taxed portion; the traced base and current threshold yield $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly reached taxable income near $50,869 but multiplied the entire amount by 5.3%. Idaho's zero-rate bracket must first be excluded from the taxed portion, producing $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model followed the right form but used an incorrect approximately $4,946 zero-tax threshold and rounded taxable income prematurely. Applying the exact 2026 threshold to $50,868.68 produces $2,435.28, not $2,433.92."
-us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly identified taxable income near $50,869 but applied 5.3% to all of it. It omitted Idaho's zero-rate bracket, whose application reduces the tax to $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an obsolete approximate 5.8% rate and an overstated taxable-income base near $52,000. The traced base is $50,868.68, and Idaho's 2026 zero-rate bracket and operative rate yield $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $8,300 standard deduction instead of $16,100, overstating taxable income by $7,800. It also applied the obsolete 5.695% rate across the full base rather than using Idaho's 2026 zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $58,597 taxable-income base reflects an incorrect deduction rather than the traced $16,100 deduction. It also used an obsolete 5.8% flat rate and omitted the zero-rate bracket."
-us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly reached taxable income near $50,869 but applied 5.3% to every taxable dollar. Idaho's zero-rate bracket excludes the initial portion from tax, yielding $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured answer was missing."
-us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached taxable income near $50,869 but multiplied the entire amount by 5.3%. Applying Idaho's zero-rate bracket before the 5.3% rate produces $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The submitted $2,878 does not follow from the model's own stated calculation of $66,969 times 5.695%, which is about $3,814. It also failed to subtract the $16,100 deduction and apply Idaho's 2026 zero-rate bracket, the steps yielding $2,435.28."
-us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored insurance premiums from the traced AGI and then used an unspecified deduction to reach its estimate. It also used an obsolete approximate 5.8% rate rather than applying the 2026 zero-rate bracket and operative rate to $50,868.68."
-us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The unexplained $1,588 answer is inconsistent with Idaho's 2026 computation on this income. Subtracting the $16,100 deduction leaves $50,868.68 taxable, and applying the zero-rate bracket and operative rate yields $2,435.28."
-us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,The model incorrectly classified Idaho's grocery credit as nonrefundable and imposed an income-based exclusion. The full-year resident qualifies for the refundable $155 grocery credit without a phase-out.
-us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model overlooked Idaho's refundable grocery credit by focusing on the absence of a state EITC and assuming income disqualified the taxpayer. The applicable grocery credit is $155 and is not reduced at these wages.
-us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model incorrectly treated the grocery credit as merely an offset against state tax whenever liability exceeds the credit. Idaho's grocery credit is recorded as a refundable credit, and this full-year resident receives $155 regardless of the amount of pre-credit liability."
-us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model misclassified Idaho's grocery credit as nonrefundable in effect and also used an obsolete $120 amount. The 2026 refundable base for this eligible resident is $155.
-us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model incorrectly netted the grocery credit against tax liability instead of including it in refundable credits. The taxpayer qualifies for the full $155 refundable grocery credit.
-us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly stated that Idaho's grocery credit is nonrefundable. For this full-year resident, it contributes $155 to state refundable credits."
-us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,The model incorrectly treated the grocery credit as only a tax offset and used income and dependent status as disqualifiers. Neither eliminates the full-year resident's $155 refundable grocery credit.
-us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model imposed an income limit that does not reduce this household's Idaho grocery credit. All 12 months qualify, producing the full $155 refundable amount."
-us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model identified the correct grocery-credit pathway but used a $120 amount instead of Idaho's 2026 base of $155. With 12 qualifying months, the full credit is $155."
-us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The zero answer omitted Idaho's refundable grocery credit. This full-year resident qualifies for all 12 months and receives $155.
-us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit but applied the wrong per-person amount. The 2026 nonelderly base is $155, not $120."
-us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used an obsolete $100 grocery-credit amount. Idaho's 2026 base for this eligible resident is $155.
-us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model found the correct refundable credit but used $120 rather than the 2026 base amount of $155. Full-year qualification leaves the $155 base unreduced.
-us,scenario_053,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model applied an obsolete $100 grocery-credit parameter. The applicable 2026 credit for the under-65 resident is $155.
-us,scenario_053,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,The model incorrectly classified Idaho's grocery credit as nonrefundable. It is included in state refundable credits and supplies the full $155 for this resident.
-us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The zero answer omitted the automatically relevant Idaho grocery credit. The stated Idaho residency and full-year facts establish 12 qualifying months and a $155 refundable credit.
-us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model looked only for child-related refundable credits and missed Idaho's grocery credit, which does not require a child. The sole resident qualifies for the full $155 amount."
-us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model identified the grocery credit correctly but used a $120 parameter. Idaho's 2026 refundable base is $155 for this nonelderly resident.
-us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model incorrectly applied an income cutoff to deny the Idaho grocery credit. The $66,969 wage amount does not phase out the credit, so all 12 qualifying months yield $155."
-us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly recognized full eligibility but used the wrong credit amount. Idaho's 2026 grocery-credit base is $155, not $120."
-us,scenario_053,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The zero answer omitted Idaho's refundable grocery credit. Full-year Idaho residence produces 12 qualifying months and a $155 credit.
-us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model correctly identified the refundable grocery credit but used an obsolete $100 amount. The 2026 nonelderly credit is $155.
-us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The zero answer failed to apply Idaho's refundable grocery credit. The resident qualifies for the entire year and receives $155.
-us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model identified the applicable grocery credit but substituted an approximate $120 amount. The exact 2026 base is $155 and no reduction applies.
-us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable state_refundable_credits output. The required output was the full-year Idaho grocery credit of $155.
-us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly applied full-year grocery-credit eligibility and the absence of monthly exclusions, but used $120 as the base. Idaho's 2026 base is $155."
-us,scenario_053,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model focused on children and property-tax-credit conditions and omitted Idaho's grocery credit, which requires neither. The single full-year resident qualifies for $155."
-us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model restricted its analysis to state EITC and child-credit pathways and missed Idaho's refundable grocery credit. A child is not required, and the resident receives $155."
-us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The zero answer omitted Idaho's refundable grocery credit. The household's sole resident qualifies for all 12 months, yielding $155."
-us,scenario_054,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model applied only a CHIP upper-income threshold and treated income below that threshold as sufficient for eligibility. It omitted the required Medicaid-ineligibility test: Child 1 qualifies for Medicaid under the OLDER_CHILD category and therefore is not eligible for CHIP.
-us,scenario_054,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model concluded that low income and lack of other coverage establish CHIP eligibility without checking Medicaid eligibility. Child 1's low income instead establishes Medicaid eligibility under the OLDER_CHILD category, which bars CHIP eligibility."
-us,scenario_054,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated being below the CHIP income ceiling and lacking employer-sponsored insurance as sufficient. It omitted that CHIP covers children who do not qualify for Medicaid, while Child 1 qualifies for Medicaid under the OLDER_CHILD category."
-us,scenario_054,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model conflated the Medicaid and CHIP income ranges, expressly counting income below the Medicaid threshold as CHIP-eligible. At this income, the age-10 child qualifies for Medicaid under the OLDER_CHILD category, so the Medicaid-exclusion rule makes CHIP eligibility false."
-us,scenario_054,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model combined Medicaid and CHIP into a single coverage threshold and labeled qualification for either program as CHIP eligibility. Child 1 specifically qualifies for Medicaid under the OLDER_CHILD category, and that qualification excludes CHIP."
-us,scenario_054,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model stated the controlling condition—CHIP applies only if the child is not eligible for Medicaid—but never evaluated it. The engine's age-and-income test places Child 1 in the Medicaid OLDER_CHILD category, so the model's own stated condition yields no CHIP eligibility."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model abandoned its correctly identified $50,869 taxable-income base and instead applied an obsolete 5.695% rate to an invented $53,000 base. The 2026 Idaho schedule applied to $50,868.68 yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used obsolete graduated Idaho brackets, an incorrect $7,200 deduction, and an invented $59 personal-exemption credit. Idaho uses the $16,100 deduction and the operative 2026 schedule, producing $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model guessed among 5.695%, 5.3%, and 4.7% rather than applying Idaho's 2026 tax schedule. It also used a $15,750 deduction instead of $16,100, so it did not compute tax on the correct $50,868.68 base."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used the obsolete 5.695% rate and a $4,673 threshold, then reported $2,867 despite its own displayed formula equaling about $2,631. Applying the 2026 schedule to $50,868.68 yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied an approximate 5.3% rate and an unsupported exemption threshold, then incorrectly treated the grocery credit as a nonrefundable offset. The operative 2026 Idaho schedule on $50,868.68 produces $2,435.28 before refundable credits."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $3,389 employer-sponsored insurance premium from wages even though the trace's AGI is $66,968.68, and it used a guessed $15,000 standard deduction. It then applied the obsolete 5.695% rate instead of the 2026 schedule."
+us,scenario_053,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first computed approximately $2,960 from its assumptions and then submitted $3,706 with no computation supporting that change. It also used a $15,000 deduction and obsolete 5.695% rate rather than the $16,100 deduction and 2026 schedule."
+us,scenario_053,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable-income figure of $48,180 instead of $50,868.68. It also treated 5.3% as a simple flat rate rather than applying the complete 2026 Idaho schedule."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied the obsolete 5.695% rate directly to an incorrect $49,980 taxable-income base. The correct base is $50,868.68 and the operative 2026 schedule yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer reflects only a rough flat-tax estimate after an unspecified standard deduction. It omitted the exact $16,100 deduction and operative 2026 Idaho schedule that produce $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an obsolete 5.692% rate and an unsupported $49,852 taxable-income estimate. Idaho taxable income is $50,868.68 and the 2026 schedule produces $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI to $63,580 by deducting the listed employer premium and then used an incorrect $49,980 taxable-income base. It compounded that error by applying the obsolete 5.8% rate."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the $3,389 employer premium from the trace's $66,968.68 AGI, reducing taxable income to $47,480 instead of $50,868.68. It also used the obsolete 5.695% rate."
+us,scenario_053,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted amount is consistent with applying the obsolete 5.695% rate to roughly $47,470 rather than applying the 2026 Idaho schedule to $50,868.68. That schedule yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model invented a deduction equal to $1,250 per $1,000 of income, creating an impossible $83,711.25 deduction and zero taxable income. Idaho allows the $16,100 standard deduction here, leaving $50,868.68 taxable."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model supplied a rough $2,050 bracket estimate without identifying or applying Idaho's 2026 schedule. Applying that schedule to $50,868.68 yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model gave an unsupported income-level estimate of $1,195 and did not compute Idaho taxable income or apply the 2026 schedule. The traced taxable income is $50,868.68, producing $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $15,750 deduction instead of $16,100 and applied a 5.3% rate above an incorrect $4,489 zero bracket. Those parameter errors produced $2,476.69 instead of the scheduled $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly approximated taxable income at $50,869 but applied 5.3% to every taxable dollar. Idaho's full 2026 schedule, including its zero-tax portion, yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used approximate values of $50,869, 5.3%, and a $4,946 zero-tax threshold. The exact 2026 parameters applied to $50,868.68 yield $2,435.28, not $2,433.92."
+us,scenario_053,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly identified approximately $50,869 of taxable income but applied 5.3% to the entire amount. It omitted the zero-tax portion embedded in Idaho's 2026 schedule, which reduces liability to $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an obsolete approximate 5.8% rate and an overstated taxable-income base of about $52,000. The correct $50,868.68 base under the 2026 Idaho schedule yields $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $8,300 standard deduction, leaving $58,669 taxable rather than $50,868.68. It then applied the obsolete 5.695% rate directly to that inflated base."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used an invented $8,522 standard deduction instead of Idaho's $16,100 deduction. Its resulting $58,447 base and obsolete 5.695% rate overstated the tax."
+us,scenario_053,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $58,597 taxable-income base, implying a deduction far below the applicable $16,100. It also applied the obsolete 5.8% rate rather than the operative 2026 schedule."
+us,scenario_053,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly identified approximately $50,869 of taxable income but applied 5.3% to the entire base. The 2026 Idaho schedule includes a zero-tax portion and produces $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the submission failed the required output contract."
+us,scenario_053,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly computed approximately $50,869 of taxable income but applied 5.3% to every dollar. Applying the complete 2026 Idaho schedule produces $2,435.28."
+us,scenario_053,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model claimed no deduction was needed and computed about $3,814, but then submitted $2,878 without a supporting calculation. Idaho subtracts the $16,100 standard deduction and applies the 2026 schedule to $50,868.68."
+us,scenario_053,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model applied 5.3% to the entire $50,869 taxable-income base and then invented a $10 Permanent Building Fund credit. The complete 2026 Idaho schedule yields $2,435.28 without that unsupported offset."
+us,scenario_053,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $3,389 employer premium from wages and used the obsolete 5.8% rate. It never explicitly applied the $16,100 Idaho deduction to the traced $66,968.68 AGI or the operative 2026 schedule."
+us,scenario_053,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model submitted $1,588 without stating any taxable-income, rate, bracket, deduction, or credit computation. The traced $50,868.68 taxable income under Idaho's 2026 schedule yields $2,435.28, so its number reflects an unsupported shortcut."
+us,scenario_053,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,The model misclassified Idaho's grocery credit as nonrefundable and imposed an inapplicable income restriction. The full-year resident receives the refundable $155 credit without an income-based reduction.
+us,scenario_053,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model overlooked Idaho's refundable grocery credit and incorrectly treated high earnings and the absence of a state EITC as dispositive. This resident qualifies for the full $155 grocery credit.
+us,scenario_053,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly netted the grocery credit against Idaho income-tax liability and counted only an excess as refundable. PolicyEngine classifies the full $155 grocery credit in state_refundable_credits regardless of the taxpayer's positive liability.
+us,scenario_053,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the grocery credit as a nonrefundable offset because the taxpayer had positive liability. The credit is refundable, and the full-year qualifying amount is $155."
+us,scenario_053,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model incorrectly applied the grocery credit solely against tax liability and concluded that no refundable amount remained. The entire $155 qualifying credit belongs in the refundable-credit output.
+us,scenario_053,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model misclassified Idaho's grocery credit as nonrefundable. For this full-year qualifying resident, it contributes $155 to refundable state credits."
+us,scenario_053,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly required an EITC supplement, dependents, or a grocery-credit excess over liability. Idaho's grocery credit itself is refundable and supplies the full $155."
+us,scenario_053,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,The model imposed an income limit on the Idaho grocery credit that does not reduce this household's 2026 credit. All 12 qualifying months produce the full $155 amount.
+us,scenario_053,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly identified the refundable grocery credit but used an outdated $120 amount. The 2026 base for this resident is $155.
+us,scenario_053,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits Idaho's grocery credit. This resident qualifies for all 12 months, yielding a refundable credit of $155."
+us,scenario_053,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used $120 as the under-65 grocery-credit amount. Idaho's applicable 2026 base is $155, with no reduction."
+us,scenario_053,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used an obsolete $100 grocery-credit parameter. The applicable 2026 full-year amount is $155.
+us,scenario_053,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model correctly found the grocery-credit pathway but applied a $120 per-person amount instead of the 2026 base of $155.
+us,scenario_053,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model applied an outdated $100 nonelderly grocery-credit amount. The 2026 full-year base is $155.
+us,scenario_053,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model misclassified Idaho's grocery credit as nonrefundable. It is the applicable refundable credit here and contributes $155.
+us,scenario_053,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model omitted the automatically applicable Idaho grocery credit from the listed facts. Full-year Idaho residency qualifies the head for the refundable $155 amount.
+us,scenario_053,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model looked only for child-related refundable credits and omitted Idaho's resident grocery credit. The single adult receives $155 without needing a child.
+us,scenario_053,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model identified the correct credit but used $120 rather than Idaho's $155 base for 2026. No phase-out or partial-year adjustment reduces that base.
+us,scenario_053,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model incorrectly disqualified the resident from the grocery credit based on $66,969 of wages. That income does not reduce the traced 2026 credit, so the full $155 applies."
+us,scenario_053,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,The model correctly found full eligibility but used a $120 grocery-credit amount. The governing 2026 base is $155.
+us,scenario_053,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The zero answer omits Idaho's refundable grocery credit. The resident qualifies for the entire year and receives $155.
+us,scenario_053,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model correctly treated the grocery credit as refundable but used an outdated $100 amount. The 2026 amount for this resident is $155.
+us,scenario_053,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,The model applied a $100 grocery-credit parameter instead of the 2026 $155 base. Full-year qualification leaves the $155 unreduced.
+us,scenario_053,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model omitted the Idaho grocery-credit eligibility pathway. The head's full-year qualification produces $155 of refundable state credits.
+us,scenario_053,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model identified the correct refundable credit but estimated its amount as $120. Idaho's 2026 base is exactly $155.
+us,scenario_053,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits.
+us,scenario_053,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly applied full-year grocery-credit eligibility but used $120 as the base. The 2026 full-year base is $155.
+us,scenario_053,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model searched for child and property-tax qualifications while omitting Idaho's grocery credit, which does not require either. This full-year resident receives $155."
+us,scenario_053,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,The model used $120 per qualifying family member rather than the applicable 2026 amount. One qualifying resident produces a $155 credit.
+us,scenario_053,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model limited refundable-credit analysis to state EITC and CTC programs and omitted Idaho's grocery credit. The credit requires no child and contributes $155 for this resident.
+us,scenario_053,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model omitted the refundable Idaho grocery credit from the household's applicable programs. Full-year qualification yields $155.
+us,scenario_054,child1_chip_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied only a CHIP upper-income threshold and concluded that income below it establishes eligibility. It omitted the Medicaid-exclusion step: the child’s low income qualifies them for Medicaid under the OLDER_CHILD category, which makes CHIP eligibility false."
+us,scenario_054,child1_chip_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated being below North Carolina’s CHIP income limit and lacking other coverage as sufficient. It failed to test Medicaid eligibility first; the 10-year-old is Medicaid-eligible under the OLDER_CHILD category and therefore cannot qualify for CHIP.
+us,scenario_054,child1_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model inferred CHIP eligibility directly from low income and the absence of employer-sponsored insurance. It omitted that low income places Child 1 in Medicaid’s OLDER_CHILD pathway, and Medicaid eligibility disqualifies the child from CHIP."
+us,scenario_054,child1_chip_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model conflated the combined Medicaid/CHIP coverage range with CHIP eligibility and expressly treated income below the Medicaid boundary as within CHIP’s eligible range. Income below that boundary makes this 10-year-old eligible for Medicaid under the OLDER_CHILD category, so the child fails CHIP’s requirement of Medicaid ineligibility."
+us,scenario_054,child1_chip_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model collapsed Medicaid and CHIP into a single eligibility outcome, stating that the child qualifies for “CHIP/Medicaid coverage.” The child’s low income specifically establishes Medicaid eligibility under the OLDER_CHILD category, and that result excludes CHIP eligibility."
+us,scenario_054,child1_chip_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model stated the controlling condition—CHIP applies only if the child is not Medicaid-eligible—but never evaluated it. Applying the condition yields Medicaid eligibility under the OLDER_CHILD category and therefore no CHIP eligibility.
+us,scenario_054,child1_chip_eligible,ox-alpha,llm_error,categorical_eligibility,False,"The model substituted “not enrolled in Medicaid” for “not eligible for Medicaid,” confusing current enrollment with categorical eligibility. Child 1 is Medicaid-eligible under the OLDER_CHILD category regardless of unlisted enrollment, and that eligibility bars CHIP."
 us,scenario_054,child1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to place Child 1 in North Carolina’s OLDER_CHILD Medicaid category and incorrectly treated unlisted citizenship, coverage, and household qualifiers as required negative evidence. The child is a dependent age 10 with MAGI at 0.58 times FPL, which satisfies the applicable child Medicaid pathway."
 us,scenario_054,child1_wic_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model invented a WIC pathway for nutritionally at-risk children ages 6–18. WIC child eligibility ends at age five, so the 10-year-old is categorically ineligible regardless of household income or the head’s WIC receipt."
 us,scenario_054,child1_wic_eligible,qwen3.8-max,llm_error,age_disability,False,"The model incorrectly treated age 10 as within WIC’s child eligibility range. Children qualify only while under age five, so Child 1 fails the categorical age requirement before the income test is relevant."
@@ -3386,32 +3560,34 @@ us,scenario_054,child2_medicaid_eligible,claude-haiku-4.5,llm_error,health_cover
 us,scenario_054,child2_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply the age-based YOUNG_CHILD Medicaid category and incorrectly demanded additional categorical indicators. At age 2 and 0.58 times FPL, Child 2 satisfies North Carolina's young-child income pathway; employer-sponsored insurance does not negate eligibility."
 us,scenario_054,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required explicit WIC receipt or an explicit statement of eligibility for Child 2. The requested output concerns eligibility rather than enrollment; age 2 satisfies WIC's categorical age rule, and the household meets the income test, yielding eligible."
 us,scenario_054,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model understated the two-child EITC as $3,995 instead of applying the 40% phase-in rate to $15,891.93 of credit earned income, which yields $6,356.77. It also omitted its own calculated refundable CTC from the submitted total."
-us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's reasoning calculated a combined estimate near $8,686 but submitted only $6,269, effectively dropping the refundable CTC and changing its EITC without a supporting computation. The required components are $6,356.77 of EITC and $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model misapplied the two-child EITC phase-in, describing a 45% rate and then assigning only about $3,000 despite $15,891.93 of credit earned income. The applicable 40% calculation yields $6,356.77, and the refundable CTC is $2,008.79."
-us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly placed the household in the EITC phase-out region and reduced the credit to about $4,700. At $15,891.93 of credit earned income, the household remains in the 40% phase-in and receives $6,356.77 of EITC plus $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model understated both components: the two-child EITC is 40% of $15,891.93, or $6,356.77, rather than $6,164, and the refundable CTC is 15% of the amount above $2,500, or $2,008.79, rather than $1,914."
-us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly calculated the $6,356.77 EITC but used a $3,000 refundable-CTC earned-income threshold. The applicable threshold is $2,500, producing $2,008.79 rather than $1,933.79."
-us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced credit earned income to $14,584 by subtracting the wrong self-employment amounts. The credit calculation uses $15,891.93, yielding $6,356.77 of EITC, and the refundable CTC uses the $2,500 threshold to yield $2,008.79."
-us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The submitted $6,867 includes only about $497 beyond its stated $6,370 EITC, so it omitted most of the $2,008.79 refundable CTC. The two components total $8,365.56."
-us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly calculated the $6,356.77 EITC but applied a $3,000 threshold to the refundable CTC. Using the applicable $2,500 threshold gives $2,008.79 and raises the total to $8,365.56."
-us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used the obsolete $3,000 ACTC threshold and an incorrect $1,000-per-child cap. The applicable $2,500 threshold produces a $2,008.79 refundable CTC, which combines with the $6,356.77 EITC."
-us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $8,291 total matches the common calculation using a $3,000 refundable-CTC threshold instead of $2,500. The correct components are $6,356.77 of EITC and $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The $8,290.56 total exactly reflects a $6,356.77 EITC plus a $1,933.79 refundable CTC calculated with a $3,000 threshold. The applicable $2,500 threshold makes the refundable CTC $2,008.79."
-us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,other,False,"The model submitted only its refundable-CTC estimate and entirely omitted the two-child EITC. The EITC is $6,356.77, and the refundable CTC must be calculated from $15,891.93 of credit earned income, yielding $2,008.79 rather than $2,190."
-us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly denied all refundable credits despite $17,100 of self-employment income and two qualifying children. Net credit earned income supports a $6,356.77 two-child EITC and a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated filing and credit-eligibility facts as missing even though the prompt directs it to assume filing and supplies the household relationship, ages, and self-employment income. Those facts establish two qualifying children and produce $6,356.77 of EITC plus $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model applied both credit formulas directly to the $17,100 gross business amount. The credit earned-income base is $15,891.93 after the applicable self-employment adjustment, producing $6,356.77 of EITC and $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used the $17,100 gross self-employment amount as earned income for both credits. Using $15,891.93 instead yields a $6,356.77 EITC and a $2,008.79 refundable CTC."
-us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used an understated and unsupported earned-income base, producing only $5,834 of EITC and $1,813 of refundable CTC. The applicable $15,891.93 base yields $6,356.77 and $2,008.79, respectively."
-us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model's reference to “one qualifying child and one young child” failed to count both dependent children in the two-child EITC category. Both ages 10 and 2 are qualifying children, and the resulting EITC is $6,356.77 before adding $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the obsolete $3,000 ACTC threshold, which understated the refundable CTC. The threshold is $2,500, so the refundable CTC is $2,008.79; its EITC also requires the precise $15,891.93 base and equals $6,356.77."
-us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model calculated EITC from the $17,100 gross business amount rather than $15,891.93 of credit earned income. It also imposed a reverted $1,000-per-child CTC cap and a $3,000 threshold instead of calculating a $2,008.79 refundable CTC above $2,500."
-us,scenario_054,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used gross self-employment receipts directly for EITC and assumed a flat $1,000 refundable CTC for each child. The calculations instead use $15,891.93 of credit earned income, yielding $6,356.77 of EITC and $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for federal_refundable_credits.
-us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model's submitted $672 contradicts its own stated combined estimate of $7,932 and omits the required credit computation. The correct component calculation produces $6,356.77 of EITC and $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly excluded specified-service-trade self-employment income from earned income for EITC and refundable CTC purposes. SSTB status concerns other tax provisions and does not disqualify these earnings; the income produces $6,356.77 of EITC and $2,008.79 of refundable CTC."
-us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model understated the EITC by using $15,522 and an incorrect credit computation, and it treated the refundable CTC as only the residual “needed” after offsetting tax. With zero pre-refundable tax, the earned-income formula produces $2,008.79 of refundable CTC, while the two-child EITC is $6,356.77."
+us,scenario_054,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It severely understated the two-child EITC and then submitted only that $3,995 estimate, omitting its own refundable CTC calculation from the total. The required components are $6,356.77 of EITC and $2,008.79 of refundable CTC."
+us,scenario_054,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"It calculated a combined refundable-credit estimate in its reasoning but submitted an EITC-only figure of $6,269. It also used inconsistent earned-income amounts and overstated the refundable CTC instead of applying 15% above $2,500 to the approximately $15,891.93 credit base."
+us,scenario_054,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"It incorrectly estimated the two-child EITC at roughly $3,000 despite identifying that the household remains in the 40% phase-in range. Applying that rate to approximately $15,891.93 yields $6,356.77, before adding $2,008.79 of refundable CTC."
+us,scenario_054,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"It incorrectly placed the household in the EITC phase-out region and reduced the credit to about $4,700. The household remains in the two-child 40% phase-in range, producing $6,356.77 of EITC, and the refundable CTC is $2,008.79."
+us,scenario_054,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It understated both components: the two-child EITC is $6,356.77, not $6,164, and the refundable CTC is $2,008.79, not $1,914. The latter applies 15% above $2,500 to earned income of approximately $15,891.93."
+us,scenario_054,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"It correctly calculated the $6,356.77 EITC but used an obsolete $3,000 refundable-CTC earnings threshold. Applying the $2,500 threshold raises the refundable CTC from $1,933.79 to $2,008.79."
+us,scenario_054,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It reduced the self-employment credit base to $14,584 by deducting too much, thereby understating both refundable credits. The applicable earned-income base is approximately $15,891.93 after the deductible half of self-employment tax, yielding $6,356.77 of EITC and $2,008.79 of refundable CTC."
+us,scenario_054,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"Its total includes only about $497 of refundable CTC after an approximately correct EITC estimate. The refundable CTC is $2,008.79 under the 15%-of-earned-income-above-$2,500 limitation."
+us,scenario_054,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"It correctly calculated the $6,356.77 EITC but used a $3,000 refundable-CTC earnings threshold. The applicable $2,500 threshold produces a $2,008.79 refundable CTC."
+us,scenario_054,federal_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It used the wrong $3,000 refundable-CTC earnings threshold and an incorrect $1,000-per-child cap. The income limitation with the applicable $2,500 threshold produces $2,008.79, which must be added to the $6,356.77 EITC."
+us,scenario_054,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"Its unexplained $8,291 total matches the common calculation using the correct EITC but a $3,000 refundable-CTC threshold. The $2,500 threshold produces $2,008.79 of refundable CTC and a total of $8,365.56."
+us,scenario_054,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"Its $8,290.56 total implies $6,356.77 of EITC plus a $1,933.79 refundable CTC calculated with a $3,000 earnings threshold. The applicable $2,500 threshold makes the refundable CTC $2,008.79."
+us,scenario_054,federal_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"It omitted the EITC entirely and submitted only its refundable-CTC estimate. Self-employment earnings qualify for the two-child EITC, which contributes $6,356.77, and the refundable CTC based on the adjusted earned-income base is $2,008.79."
+us,scenario_054,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"It denied refundable credits despite the stated self-employment income and two qualifying children. Those facts generate a $6,356.77 two-child EITC and a $2,008.79 refundable CTC."
+us,scenario_054,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"It wrongly required additional explicit filing or eligibility indicators even though the prompt directs assuming filing and supplies the income and qualifying-child facts. The household qualifies for $6,356.77 of EITC and $2,008.79 of refundable CTC."
+us,scenario_054,federal_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It applied both credit formulas directly to the $17,100 gross self-employment profit. The credit base is approximately $15,891.93 after the deductible half of self-employment tax, producing $6,356.77 of EITC and $2,008.79 of refundable CTC."
+us,scenario_054,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It used the full $17,100 self-employment profit as earned income for both credits and obtained $6,840 plus $2,190. After the deductible half of self-employment tax, the applicable base is approximately $15,891.93, yielding $6,356.77 and $2,008.79."
+us,scenario_054,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It understated both the EITC and refundable CTC through an incorrect earned-income base. The applicable base of approximately $15,891.93 yields $6,356.77 of EITC and $2,008.79 of refundable CTC."
+us,scenario_054,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"Its reference to “one qualifying child and one young child” failed to treat both dependent children as qualifying children in the two-child EITC calculation. Both children count, producing $6,356.77 of EITC, with another $2,008.79 from refundable CTC."
+us,scenario_054,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"It used the obsolete $3,000 refundable-CTC threshold and a slightly understated earned-income base. With approximately $15,891.93 of earned income and the $2,500 threshold, the components are $6,356.77 and $2,008.79."
+us,scenario_054,federal_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"It used a $3,000 refundable-CTC threshold rather than $2,500 and slightly understated the credit base. The correct refundable CTC is $2,008.79 and the two-child EITC is $6,356.77."
+us,scenario_054,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It applied the EITC to gross self-employment profit rather than the approximately $15,891.93 base after the deductible half of self-employment tax. It also imposed a reverted $1,000-per-child refundable-CTC cap instead of calculating the $2,008.79 income-limited refundable amount."
+us,scenario_054,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It applied the EITC directly to the $17,100 gross self-employment profit and assumed a flat $1,000 refundable CTC for each child. The adjusted earned-income base yields $6,356.77 of EITC, while the refundable CTC formula yields $2,008.79."
+us,scenario_054,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for federal_refundable_credits.
+us,scenario_054,federal_refundable_credits,minimax-m3,llm_error,other,False,"It submitted $672 even though its own stated component estimates totaled $7,932, so the final value does not implement its reasoning. Its component reasoning also used the wrong earned-income base and treated the refundable CTC as automatically equal to the per-child cap instead of applying the earned-income limitation."
+us,scenario_054,federal_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"It used the full $17,100 self-employment profit as the earned-income base, producing $6,840 of EITC and $2,190 of refundable CTC. The base after the deductible half of self-employment tax is approximately $15,891.93, producing $6,356.77 and $2,008.79."
+us,scenario_054,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"It wrongly treated SSTB self-employment income as excluded from earned income for the EITC and refundable CTC. SSTB status concerns the qualified-business-income deduction, not whether net self-employment earnings support these credits; the household receives both refundable components."
+us,scenario_054,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It incorrectly limited refundable CTC to the portion “needed” after the CTC offsets regular tax, even though zero regular tax leaves the income-limited refundable portion available rather than reducing it to $375. It also used an understated earned-income base and EITC; the correct components are $6,356.77 and $2,008.79."
 us,scenario_054,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required an explicit free-meal indicator or a stated SNAP amount instead of calculating eligibility from the household facts. Income at 63% of the federal poverty guideline independently satisfies the free-meal threshold of 130%, and SNAP participation also establishes categorical eligibility."
 us,scenario_054,head_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated the head as a childless adult even though the household includes the head’s two children. It therefore omitted the parent/caretaker Medicaid pathway, under which the head’s MAGI of 0.58 times FPL qualifies."
 us,scenario_054,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model searched only for pregnancy, disability, and similar indicators and failed to recognize that living with two dependent children places the head in the parent/caretaker category. That category and the head’s MAGI of 0.58 times FPL establish eligibility without an asset test."
@@ -3441,66 +3617,69 @@ us,scenario_054,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,Fals
 us,scenario_054,self_employment_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's own formula produces approximately $2,416, but it submitted $1,596 without a valid computation connecting the two amounts. Excluding Additional Medicare Tax does not reduce the ordinary 12.4% Social Security plus 2.9% Medicare SECA liability, which equals $2,416.15."
 us,scenario_054,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly described a result near $2,416.18 but submitted the inconsistent amount $2,419.95. Exact arithmetic using the unrounded base gives $15,791.85 × 15.3% = $2,416.15305, rounded to $2,416.15."
 us,scenario_054,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the deduction for one-half of self-employment tax from the SECA liability and then produced a final number inconsistent with its intermediate figures. That deduction reduces adjusted gross income for income-tax purposes; it does not reduce self-employment tax, which remains $17,100 × 92.35% × 15.3% = $2,416.15."
-us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model projected an unsupported fixed $820 maximum allotment and applied it to all twelve months. The applicable maximum changes from $785 to $803, so the monthly amounts must be aggregated to $6,125.69."
-us,scenario_054,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model imposed a conventional SNAP resource limit on the $14,000 bank balance. This household passes North Carolina categorical eligibility through TANF non-cash assistance, so that asset shortcut does not disqualify it."
-us,scenario_054,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model first derived about $6,456 and then replaced it with $7,152 based on an unexplained COLA adjustment. It also used one estimated maximum allotment instead of aggregating the $785-$803 monthly maxima across 2026."
-us,scenario_054,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model correctly derived a benefit near $508 per month, then replaced it with an unsupported $707 monthly figure. The traced monthly benefits are about $505-$525 and total $6,125.69, not $8,484."
-us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used an unexplained estimate of $583 per month after identifying the relevant income deductions. Net income near $931 and the applicable $785-$803 maxima produce about $505-$525 per month, which must be summed across the year."
-us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by 40% for business costs even though no such expenses were listed, driving countable income down to $456 per month. It then submitted $5,532 despite repeatedly calculating $7,548, so its final value follows neither its own deduction calculation nor the applicable monthly allotments."
-us,scenario_054,snap,claude-sonnet-5,llm_error,thresholds_rates,False,The model stated that roughly $768 minus a $280 contribution produced only $350 per month; that subtraction is wrong. It also used a single estimated maximum rather than the monthly $785-$803 values.
-us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model fixed the maximum allotment at $781 for every month. PolicyEngine applies monthly maxima of $785-$803 and aggregates the resulting benefits, producing $6,125.69."
-us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an obsolete or unsupported $766 maximum allotment for every month. The 2026 monthly maxima are $785-$803, so subtracting the roughly $278-$279 contribution month by month yields $6,125.69."
-us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $3,720 is an unsupported coarse estimate. The traced calculation uses net monthly income near $931, subtracts a 30% contribution from each month's $785-$803 maximum, and totals $6,125.69."
-us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The answer implies the same fixed-allotment calculation as the $5,800.80 estimate, rather than applying the changing 2026 monthly maxima. Month-by-month aggregation of benefits near $505-$525 yields $6,125.69."
-us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $747 maximum allotment, below every applicable 2026 monthly maximum of $785-$803. That stale threshold understates each month's benefit and therefore the annual total."
-us,scenario_054,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $7,548 answer exceeds the traced result by using an overstated benefit schedule or understated contribution. Net income near $931 produces a $278-$279 contribution, and the $785-$803 monthly maxima yield $6,125.69."
-us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model supplied a generic estimate without applying the actual 2026 monthly maximum-allotment schedule. The correct month-specific benefits are about $505-$525 and sum to $6,125.69."
-us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"The model estimated a round $9,600 annual maximum instead of using the month-specific $785-$803 allotments. Its deduction calculation was close, but the incorrect maximum schedule raised the result to $6,317."
-us,scenario_054,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The $8,868 estimate treats the household as receiving close to the maximum allotment despite countable net income near $931 per month. SNAP subtracts a 30% contribution of about $278-$279 each month, leaving only about $505-$525."
-us,scenario_054,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the explicitly supplied household size and $17,100 of self-employment income and therefore never performed the eligibility or benefit calculation. The household passes categorical eligibility through TANF non-cash assistance and receives a positive benefit."
-us,scenario_054,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model made a direct subtraction error: $785 minus 30% of $931 is about $506, not $575. It also annualized one month's maximum instead of aggregating the $785-$803 maxima across 2026."
-us,scenario_054,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model disqualified the household under an asset limit. North Carolina categorical eligibility through TANF non-cash assistance allows the household to pass despite the $14,000 bank balance."
-us,scenario_054,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model correctly identified categorical eligibility, net income near $931, and a benefit near $506 for a $785-maximum month. It incorrectly multiplied that single-month amount by twelve instead of applying the later $803 maximum and aggregating all monthly values."
-us,scenario_054,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model correctly calculated about $506 using the $785 maximum, but treated that amount as constant for twelve months. The maximum rises to $803 for part of the year, so the monthly results must be aggregated to $6,125.69."
-us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared both income and assets excessive. Gross and net income are well below their applicable limits, and TANF non-cash categorical eligibility prevents the $14,000 bank balance from disqualifying the household."
-us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used an annual maximum of $9,192, equivalent to a fixed $766 monthly allotment. The applicable monthly maxima are $785-$803, so its stale maximum schedule understates the benefit."
-us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an estimated fixed $811 monthly maximum, above the applicable $785-$803 range. That inflated schedule produces $6,370 instead of the month-by-month total."
-us,scenario_054,snap,inkling,llm_error,asset_resource,False,"The model applied a resource limit to the $14,000 bank account and stopped the calculation. The household passes categorical eligibility through TANF non-cash assistance, so the assets do not eliminate SNAP eligibility."
-us,scenario_054,snap,kimi-k2.6,llm_error,asset_resource,False,"The model applied the ordinary resource test while omitting North Carolina's categorical-eligibility pathway. TANF non-cash categorical eligibility allows this household to qualify despite the $14,000 bank balance."
-us,scenario_054,snap,kimi-k3,llm_error,period_annualization,False,"The model correctly derived net income of $931 and correctly disregarded the bank balance, but used a fixed $783 maximum for all twelve months. The actual monthly maximum ranges from $785 to $803, requiring month-by-month aggregation."
-us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"The model compared $17,100 with a roughly $26,000 limit and then reversed the inequality: $17,100 is below that figure, not above it. The household also passes categorical eligibility through TANF non-cash assistance."
-us,scenario_054,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded the maximum benefit despite the household's positive countable income. SNAP subtracts 30% of net income near $931 each month, and the applicable maximum is $785-$803 rather than $934."
-us,scenario_054,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model used an unexplained $1,579 earnings figure instead of the stated $1,425 monthly self-employment income and incorrectly concluded that net income exceeded the limit. The household's gross and net income pass the tests, and it also qualifies categorically through TANF non-cash assistance."
+us,scenario_054,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model projected an approximately $820 maximum monthly allotment instead of applying the monthly 2026 values of $785 to $803. With net income near $931, the monthly results must be calculated under the applicable parameters and summed to $6,125.69."
+us,scenario_054,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model imposed a conventional SNAP asset limit on the $14,000 bank balance. This household passes categorical eligibility through TANF non-cash assistance, so those assets do not eliminate its $6,125.69 benefit."
+us,scenario_054,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model improperly reduced SNAP self-employment income by half of self-employment tax and then replaced its own $6,456 calculation with an unsupported $7,152 COLA adjustment. SNAP net income is about $931 per month here, and the monthly allotments must be summed under the actual $785-to-$803 maxima."
+us,scenario_054,snap,claude-opus-4.8,llm_error,other,False,"The model correctly derived an allotment near $508 per month and an annual estimate near $6,096, then substituted an unexplained $707 monthly amount to report $8,484. That refinement does not follow from its stated net income or maximum allotment; the monthly engine results sum to $6,125.69."
+us,scenario_054,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated $583 per month without reconciling that figure to its stated maximum near $800 and the 30% contribution from net income. Applying the actual monthly maxima of $785 to $803 to net income near $931 yields annual SNAP of $6,125.69."
+us,scenario_054,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied both a 40% self-employment business-cost deduction and a 20% earned-income deduction, reducing countable income to $456 per month instead of about $931. Its submitted $5,532 also contradicts every annual amount in its own reasoning, none of which follows the engine's monthly calculation."
+us,scenario_054,snap,claude-sonnet-5,llm_error,other,False,"The model states a maximum near $768 and a contribution near $280, whose difference is about $488 rather than $350. It therefore made an internal subtraction error; the actual varying monthly maxima and contribution produce $6,125.69 annually."
+us,scenario_054,snap,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model held the maximum allotment at $781 for all twelve months. PolicyEngine uses the applicable 2026 monthly maxima of $785 to $803, so the monthly benefits sum to $6,125.69 rather than $5,988."
+us,scenario_054,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a stale $766 maximum allotment for every month. The applicable monthly maxima range from $785 to $803, and their benefits after the 30% contribution sum to $6,125.69."
+us,scenario_054,snap,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied only an unsupported generic estimate of $3,720. The correct computation uses monthly net income near $931, subtracts 30% of it from monthly maxima of $785 to $803, and totals $6,125.69."
+us,scenario_054,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The answer is consistent with using the stale $766 maximum allotment underlying the same $5,800.80 calculation. Applying the 2026 monthly maxima of $785 to $803 and summing each month yields $6,125.69."
+us,scenario_054,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a $747 maximum allotment, below every applicable 2026 monthly maximum of $785 to $803. Its net-income calculation was close, but the obsolete maximum understated annual SNAP."
+us,scenario_054,snap,gemini-3.6-flash,llm_error,other,False,"The model gave no computation supporting $7,548. The engine calculation subtracts 30% of monthly net income near $931 from maxima of $785 to $803 and sums twelve monthly benefits to $6,125.69."
+us,scenario_054,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model gave no parameters supporting $5,820. The applicable monthly maxima of $785 to $803, reduced by the 30% contribution on net income near $931, produce $6,125.69."
+us,scenario_054,snap,glm-5.2,llm_error,thresholds_rates,False,"The model approximated the annual maximum allotment as $9,600 instead of aggregating the applicable monthly maxima. The exact month-specific maximum allotments and expected contributions yield $6,125.69, not $6,317."
+us,scenario_054,snap,gpt-5.4-mini,llm_error,other,False,"The model treated the household as having nearly maximum benefits without computing the 30% contribution from net income near $931. That contribution must be subtracted from each monthly maximum, producing $6,125.69 rather than $8,868."
+us,scenario_054,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored facts explicitly supplying household size, $17,100 of self-employment income, and $14,000 of assets. Those facts establish low income, while TANF non-cash categorical eligibility prevents the assets from disqualifying the household, resulting in positive SNAP."
+us,scenario_054,snap,gpt-5.5,llm_error,other,False,"The model's arithmetic is wrong: $785 minus 30% of $931 is about $506, not $575. It also held the maximum at $785 instead of applying the later $803 value, so the correct monthly aggregation is $6,125.69."
+us,scenario_054,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model disqualified the household under an asset limit. TANF non-cash categorical eligibility makes the $14,000 bank balance non-disqualifying for this SNAP determination."
+us,scenario_054,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model used approximately $506 for all twelve months, capturing only the $785 maximum-allotment period. PolicyEngine also applies the later monthly maximum of $803, and summing the month-specific benefits produces $6,125.69."
+us,scenario_054,snap,gpt-5.6-terra,llm_error,period_annualization,False,"The model annualized the $506 benefit calculated from a fixed $785 maximum. The maximum changes to $803 during the calendar year, so the twelve month-specific allotments sum to $6,125.69."
+us,scenario_054,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared both income and assets excessive. Monthly gross income of $1,425 is far below the stated poverty-guideline thresholds, and TANF non-cash categorical eligibility prevents the $14,000 balance from disqualifying the household."
+us,scenario_054,snap,grok-4.5,llm_error,thresholds_rates,False,"The model used an annual maximum allotment of $9,192, equivalent to a stale fixed $766 monthly maximum. PolicyEngine applies monthly maxima of $785 to $803, raising the correctly aggregated benefit to $6,125.69."
+us,scenario_054,snap,grok-4.6,llm_error,thresholds_rates,False,"The model used a fixed annual maximum allotment of $9,216, or $768 per month. The applicable 2026 monthly maxima are $785 to $803, and their month-specific benefit calculations sum to $6,125.69."
+us,scenario_054,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed an approximately $811 maximum for every month, exceeding the applicable $785-to-$803 range. Using each month's actual maximum and expected contribution produces $6,125.69 rather than $6,370."
+us,scenario_054,snap,inkling,llm_error,asset_resource,False,"The model imposed a SNAP resource limit on the $14,000 bank balance. The household qualifies categorically through TANF non-cash assistance, so the balance does not bar SNAP."
+us,scenario_054,snap,kimi-k2.6,llm_error,asset_resource,False,"The model applied the ordinary resource limit for a household without an elderly or disabled member. It missed the TANF non-cash categorical-eligibility pathway, under which the $14,000 balance does not disqualify this household."
+us,scenario_054,snap,kimi-k3,llm_error,thresholds_rates,False,"The model used a fixed $783 monthly maximum and rounded the expected contribution to $280 throughout the year. PolicyEngine uses monthly maxima of $785 to $803 and the applicable month-specific contribution calculation, totaling $6,125.69."
+us,scenario_054,snap,minimax-m3,llm_error,thresholds_rates,False,"The model compared $17,100 with an annual 130%-of-poverty threshold of about $26,000 and then reversed the comparison: $17,100 is below $26,000. The household also qualifies categorically through TANF non-cash assistance, so its SNAP benefit is positive."
+us,scenario_054,snap,ox-alpha,llm_error,period_annualization,False,"The model correctly computed about $505.70 under a $785 monthly maximum but applied that value to all twelve months. The maximum rises to $803 for part of the calendar year, and the month-specific benefits sum to $6,125.69."
+us,scenario_054,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model awarded a supposed maximum benefit despite the household's positive countable net income near $931 per month. SNAP subtracts a 30% expected contribution from monthly maxima of $785 to $803, so the household receives $6,125.69 rather than a maximum allotment."
+us,scenario_054,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model used an unexplained $1,579 earnings figure instead of the stated $1,425 monthly self-employment income and incorrectly found income above the limit. The household's gross and net income pass the relevant tests, and TANF non-cash categorical eligibility supplies an additional eligibility pathway."
 us,scenario_054,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly calculated North Carolina taxable income as $0 and tax as $0, then submitted $245 anyway. Its numeric output directly contradicts every computation step and conclusion in its explanation."
 us,scenario_054,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model first derived $0 taxable income using the head-of-household standard deduction, then switched without basis to single filing status and ultimately submitted $596 without showing any additions or subtractions that create that liability. It discarded the zero tax base and invented a positive North Carolina tax unsupported by its own arithmetic."
 us,scenario_054,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's $142 answer implies that it retained a positive North Carolina taxable-income base after the standard deduction. The applicable computation yields no positive taxable income, so applying the flat rate must produce $0 rather than $142."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived essentially the correct gross income, $1,123 senior deduction, and $136,002 taxable income, but then used estimated bracket cutoffs of $12,400, $50,400, and $105,700. Applying the enacted 2026 brackets to $136,002.41 yields $25,238.58, not $25,479."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an invented $23,750 standard deduction, initially treated all Social Security as gross income, and then submitted $14,829 despite its own bracket calculation producing roughly $25,000. The required deductions total $20,273.45 and leave $136,002.41 taxable, producing $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model treated the $6,000 senior deduction as phaseout-free even though $156,275.86 of income reduces it to $1,123.45. It then discarded its own $24,431 computation and submitted $13,802 without any deduction or credit that produces that reduction."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the separate income-phased senior deduction of $1,123.45 and used an estimated $18,600 standard deduction instead of the applicable $18,150 amount. The total allowable deduction is $20,273.45, leaving $136,002.41 taxable."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included personal auto-loan interest in itemized deductions and omitted the charitable-contribution floor and the $1,123.45 phased senior deduction. It also asserted that about $138,526 of taxable income generated only $24,196, which does not follow from the applicable 2026 brackets."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used projected parameters and stopped after the ordinary age-based standard deduction, omitting the separate $1,123.45 senior deduction. The applicable deductions total $20,273.45 rather than $17,700, so taxable income is $136,002.41 rather than $138,576."
-us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model misstated the senior-deduction phaseout and alternately used about $4,000 and $8,175 instead of the $1,123.45 remaining deduction. It then reduced its own $24,788 tax calculation to $18,033 without identifying any allowable credit or further deduction."
-us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model assumed expiration-era law, restoring a personal exemption and the 15%, 25%, and 28% brackets while assigning an implausibly low standard deduction. The applicable 2026 structure instead supplies $20,273.45 of deductions and retains the 10%, 12%, 22%, and 24% brackets at this income."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the tax system reverted to pre-TCJA rules, claimed a personal exemption, and applied 10%, 15%, 25%, and 28% rates. The applicable 2026 rules retain the current bracket structure and yield $25,238.58 on $136,002.41 of taxable income."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model included the full $5,742 Social Security benefit instead of the taxable $4,880.71 and used a $16,800 deduction rather than total deductions of $20,273.45. Its $17,296 answer also does not result from applying the progressive 2026 rates to the taxable income implied by its explanation."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model improperly restored a personal exemption and treated $16,484 of unadjusted itemized deductions as controlling. The applicable deduction comparison leads to $20,273.45 of deductions, and the enacted 2026 brackets produce $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed post-TCJA-expiration rules, claimed a $5,050 personal exemption, and applied the obsolete higher-rate bracket schedule. The applicable deduction and bracket rules leave $136,002.41 taxable and generate $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model recognized the taxable Social Security inclusion but accounted only generically for the standard deduction and age allowance, omitting the exact $1,123.45 income-phased senior deduction. The deduction total must be $20,273.45 before applying the 2026 brackets."
-us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model stopped at the ordinary standard deduction for a filer age 65 or older and omitted the separate $1,123.45 phased senior deduction. That omission overstates taxable income above the required $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $17,450 standard deduction and omitted the $1,123.45 phased senior deduction. The applicable total deduction is $20,273.45, not $17,450, and the correct taxable income is $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model invented a rule under which the standard deduction is reduced by excess itemized deductions; no such netting step applies. The taxpayer receives $20,273.45 of total deductions, and applying the brackets to $136,002.41 cannot produce $16,348."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated ordinary deductions as offsetting essentially all $156,275.86 of gross income. They reduce income by only $20,273.45, leaving $136,002.41 taxable and a substantial federal liability."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"The model submitted a zero placeholder instead of performing the requested estimate. The supplied facts determine $156,275.86 of gross income, $20,273.45 of deductions, and $25,238.58 of tax."
-us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model cited a senior-enhanced standard deduction but supplied no computation supporting $24,379. The applicable deduction is $20,273.45, leaving $136,002.41 taxable; the 2026 brackets then yield $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model included the full $5,742 Social Security benefit rather than $4,880.71 and used an unspecified $17,000 deduction, producing $140,137 of taxable income. It then understated the tax on even that overstated taxable income; the required taxable income is $136,002.41 and the tax is $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a reversion to pre-TCJA law, restored a personal exemption, and applied 10%, 15%, 25%, and 28% brackets. The applicable 2026 deduction and bracket regime produces $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an expiration-era $9,650 standard deduction and obsolete 15%, 25%, and 28% brackets. It also failed to subtract the $1,123.45 phased senior deduction; total deductions are $20,273.45 and taxable income is $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model incorrectly applied pre-TCJA-expiration rules, including a personal exemption and the 15%, 25%, and 28% rate schedule. Under the applicable 2026 rules, deductions total $20,273.45 and the tax is $25,238.58."
-us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $17,400 age-adjusted standard deduction and omitted the separate $1,123.45 phased senior deduction. The applicable total deduction is $20,273.45, reducing taxable income to $136,002.41 rather than about $138,876."
-us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model misstated AGI, invented mortgage interest from the mortgage balance despite the instruction that unlisted amounts are zero, and then subtracted unspecified nonrefundable credits until the liability reached zero. No such credits apply; gross income is $156,275.86 and taxable income is $136,002.41."
-us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model included only $2,045 of Social Security even though the high provisional income makes $4,880.71 taxable, and it treated medical expenses below the 7.5% floor as itemized deductions. It also omitted the ordinary age-based and separate phased senior deductions that bring total deductions to $20,273.45."
-us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model gave no derivation, and $30,253 is inconsistent with the applicable calculation: $156,275.86 of gross income minus $20,273.45 of deductions leaves $136,002.41 taxable. Applying the 2026 brackets to that amount yields $25,238.58, so its answer reflects an incorrect rate or taxable-income base."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived essentially the correct gross income, deductions, and taxable income, but used estimated bracket cutoffs of $12,400, $50,400, and $105,700. The enacted 2026 brackets applied to $136,002.41 yield $25,238.58."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model's final $14,829 contradicts its own calculation of roughly $25,392 and is unsupported by any tax computation. It also initially included all Social Security in gross income and used an erroneous $23,750 deduction instead of taxable Social Security of $4,880.71 and total deductions of $19,273.45."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the $6,000 senior deduction as phaseout-free even though it is reduced to $1,123.45 at this income. Its own bracket calculation still produced $24,431, then it replaced that result with an unexplained $13,802."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the separate income-tested senior deduction of $1,123.45 and used an estimated $18,600 age-adjusted standard deduction. The applicable deductions total $19,273.45, leaving taxable income of $136,002.41."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model neither applied the $1,123.45 phased senior deduction nor performed a bracket calculation consistent with its stated taxable income. Taxing even its asserted $138,526 under the applicable progressive rates does not produce $24,196."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used projected standard-deduction amounts and omitted the $1,123.45 income-tested senior deduction. The correct deduction total is $19,273.45 rather than $17,700, producing $136,002.41 of taxable income."
+us,scenario_055,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model inconsistently described the senior deduction as both approximately $4,000 and $8,175, while the phaseout leaves exactly $1,123.45. Its displayed arithmetic produced $24,788, which it then replaced with the unsupported value $18,033."
+us,scenario_055,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, restored a personal exemption, and applied 10%, 15%, 25%, and 28% brackets. The applicable 2026 rules retain the current rate structure and provide a $19,273.45 deduction without a personal exemption."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed that pre-TCJA rules return in 2026, adding a personal exemption and applying 15%, 25%, and 28% brackets. The applicable 2026 calculation uses $136,002.41 of taxable income under the 10%, 12%, 22%, and 24% brackets."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $5,742 Social Security benefit as gross income and used an unsupported $16,800 age-based deduction. Gross income includes only $4,880.71 of taxable Social Security, and deductions total $19,273.45."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model itemized and claimed a personal exemption under an assumed post-TCJA-expiration regime. The governing calculation instead takes the $18,150 standard-and-age deduction plus the $1,123.45 phased senior deduction and applies the enacted 2026 brackets."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,050 personal exemption and post-TCJA-expiration brackets. The applicable 2026 rules use no personal exemption and tax $136,002.41 under the 10%, 12%, 22%, and 24% rate schedule."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model used only the ordinary standard deduction and age allowance. It omitted the additional senior deduction after phaseout, which is $1,123.45 and reduces taxable income to $136,002.41."
+us,scenario_055,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model stopped at the ordinary standard deduction for a filer age 65 or older and omitted the separate $1,123.45 income-tested senior deduction. Total deductions are $19,273.45, not merely the age-adjusted standard deduction."
+us,scenario_055,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $17,450 standard-and-age deduction and omitted the $1,123.45 phased senior deduction. The applicable combined deduction is $19,273.45, leaving $136,002.41 taxable."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model invented a reduction of the standard deduction by excess itemized deductions; itemized and standard deductions are alternatives, not offsets against one another. The greater base deduction is $18,150, followed by the $1,123.45 senior deduction."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated deductions and nonexistent credits as eliminating tax on more than $156,000 of gross income. After $19,273.45 of deductions, $136,002.41 remains taxable and generates $25,238.58 before refundable credits."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,missing_output,False,"The model submitted a zero placeholder instead of performing the requested calculation. The supplied facts yield $156,275.86 of gross income, $136,002.41 of taxable income, and $25,238.58 of tax."
+us,scenario_055,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model gave no derivation for its senior-enhanced deduction, and its $24,379 answer reflects an excessive reduction of taxable income. The income-tested senior deduction is only $1,123.45, making total deductions $19,273.45."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model included the entire $5,742 Social Security benefit instead of the taxable $4,880.71 and left $140,137 taxable after an undersized deduction. It then applied a rate computation inconsistent with that taxable income; the proper taxable income is $136,002.41 and its tax is $25,238.58."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a 2026 TCJA sunset, restored a personal exemption, and used pre-TCJA brackets. The applicable calculation uses the $18,150 standard-and-age deduction plus a $1,123.45 phased senior deduction under the enacted 2026 rate schedule."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied TCJA-sunset rules, including a $5,300 personal exemption and 10%, 15%, 25%, and 28% brackets. The applicable 2026 rules instead yield $136,002.41 of taxable income taxed at the current bracket structure."
+us,scenario_055,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed post-TCJA brackets and an erroneously low standard deduction, then failed to subtract the personal exemption its assumed regime would require. The applicable deduction is $19,273.45 and the enacted 2026 rates produce $25,238.58."
+us,scenario_055,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and the pre-TCJA 10%, 15%, 25%, and 28% brackets. The applicable 2026 computation takes $19,273.45 of deductions and taxes $136,002.41 under the enacted schedule."
+us,scenario_055,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $17,400 age-adjusted standard deduction and omitted the separate $1,123.45 senior deduction after phaseout. The combined deduction is $19,273.45, reducing taxable income to $136,002.41."
+us,scenario_055,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model misstated AGI, invented mortgage interest from the mortgage balance despite the instruction that unlisted amounts are zero, and invented nonrefundable credits that eliminate its asserted tax. No such credits apply, and the correct taxable income is $136,002.41."
+us,scenario_055,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model calculated only $2,045 of taxable Social Security instead of $4,880.71 despite provisional income far above the upper threshold. It also treated medical expenses below the 7.5% AGI floor as deductible and omitted the $1,123.45 phased senior deduction."
+us,scenario_055,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The submitted value is inconsistent with taxing the traced $136,002.41 of taxable income under the applicable 2026 brackets. Those brackets yield $25,238.58, so $30,253 reflects an incorrect rate schedule or taxable-income base."
 us,scenario_055,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model identified the correct employee rates and wage base but rounded the computed liability to $6,400 instead of returning the component-level result. Applying those rates to $83,635 yields $5,185.38 of Social Security tax plus $1,212.71 of Medicare tax, totaling $6,398.09."
 us,scenario_055,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model’s submitted $10,534 contradicts its stated computation of 6.2% Social Security plus 1.45% Medicare on $83,635. Its own stated formula yields $6,398.09, so it made an arithmetic or final-answer transcription error."
 us,scenario_055,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model used the combined employer-and-employee rates of 12.4% for Social Security and 2.9% for Medicare even though the requested output includes only employee-side payroll tax. It then invented a partial nontaxability adjustment unsupported by the facts instead of calculating the employee components of $5,185.38 and $1,212.71."
@@ -3527,99 +3706,104 @@ us,scenario_056,self_employment_tax,grok-4.3,llm_error,payroll_tax_base,False,"T
 us,scenario_056,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax output or explanation, violating the required output contract."
 us,scenario_056,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model reversed the $400 threshold rule: $4,340 of self-employment income is above, not below, $400. Applying the 92.35% net-earnings adjustment and 15.3% combined rate yields $613.22."
 us,scenario_056,self_employment_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly declared $4,340 below the $400 self-employment-tax threshold. Because the threshold is exceeded, the 92.35% net-earnings base is taxed at 15.3%, producing $613.22."
-us,scenario_056,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model substituted a $744 federal shelter-deduction cap and its own maximum-allotment parameters for the PolicyEngine values, reducing net income to $462 instead of $805.03. That produced a $159 formula allotment and bypassed the applicable $95 minimum."
-us,scenario_056,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated $17,839 of annual income as exceeding the limit without converting it to the $1,486.54 monthly gross-income figure used by SNAP, and it treated an owner-occupied home's mortgage balance as a disqualifying asset. The household passes the income and asset tests, and allowable deductions reduce net income to $805.03."
-us,scenario_056,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model applied only the earned-income and standard deductions and omitted the $395.49 excess-shelter deduction. PolicyEngine therefore reaches $805.03 of net income and a $63.18 formula benefit, which is raised to the $95 monthly minimum."
-us,scenario_056,snap,claude-opus-4.8,llm_error,asset_resource,False,The model treated homeownership and financial assistance as evidence of disqualifying assets even though the household passes the SNAP asset test. It also failed to complete the deduction calculation that yields $805.03 of net income and triggers the $95 minimum allotment.
-us,scenario_056,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model compared the $17,839 annual total directly with SNAP limits and omitted the allowable standard, earned-income, and excess-shelter deductions. Monthly gross income is $1,486.54, and deductions reduce net income to $805.03."
-us,scenario_056,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model stopped after the earned-income and standard deductions and used $1,216 as net income, omitting the $395.49 excess-shelter deduction. PolicyEngine's $805.03 net income produces $63.18 before application of the $95 monthly minimum."
-us,scenario_056,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model asserted that $1,486.54 of monthly gross income exceeds the applicable gross-income limit, despite that amount being only 1.12 times the $1,330 poverty guideline. It also invoked unspecified asset-like resources instead of applying the passed asset test and the deductions that yield $805.03 net income."
-us,scenario_056,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model inferred hours worked from annual self-employment income and imposed an immediate ABAWD disqualification. PolicyEngine's work-requirement test is passed, so the household remains eligible and receives the $95 monthly minimum."
-us,scenario_056,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $672 shelter deduction, reducing net income to $544.25 instead of PolicyEngine's $805.03. It consequently calculated a formula allotment above the minimum rather than applying the $95 monthly floor."
-us,scenario_056,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly classified $1,486.54 of monthly gross income as above the SNAP threshold. That income is 1.12 times the $1,330 poverty guideline and passes the gross-income test."
-us,scenario_056,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used a maximum capped shelter deduction that drove net income to about $489 instead of applying PolicyEngine's $395.49 excess-shelter deduction. The correct net income is $805.03, making the formula amount $63.18 and the payable amount the $95 minimum."
-us,scenario_056,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the household as entitled to the maximum allotment after shelter deductions, effectively reducing countable net income to zero. PolicyEngine retains $805.03 of net income, yielding $63.18 before the $95 minimum is applied."
-us,scenario_056,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $140.72 monthly amount implies a lower net-income figure or different allotment parameters than the trace. PolicyEngine uses $805.03 of net income, calculates $63.18, and then pays the $95 monthly minimum."
-us,scenario_056,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model awarded the maximum allotment merely because income and shelter deductions established eligibility. Eligibility does not erase countable income: $805.03 remains, producing a subminimum $63.18 formula allotment and a $95 payment."
-us,scenario_056,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model imposed an estimated $733 federal shelter cap instead of PolicyEngine's $395.49 excess-shelter deduction, reducing net income to $471.25. PolicyEngine's $805.03 net income produces a formula amount below the applicable $95 minimum."
-us,scenario_056,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model defaulted to zero instead of using the complete household facts supplied in the prompt. Those facts yield $1,486.54 gross income, $674.49 of allowable deductions, $805.03 net income, and the $95 monthly minimum."
-us,scenario_056,snap,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model required unspecified additional low-income indicators and ignored the supplied income and shelter facts. The listed facts establish eligibility and deductions that reduce net income to $805.03, triggering the $95 minimum allotment."
-us,scenario_056,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly recognized that the formula benefit falls below the minimum but used a $23 federal minimum instead of New Jersey's applicable $95 monthly minimum in PolicyEngine. The annual amount is therefore $95 times 12, not $23 times 12."
-us,scenario_056,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model left net income high by failing to apply the full $674.49 of allowable deductions, including $395.49 of excess shelter expense. PolicyEngine obtains $805.03 net income and raises the resulting $63.18 allotment to $95."
-us,scenario_056,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $159 monthly estimate reflects an oversized capped shelter deduction that reduced net income far below $805.03. With PolicyEngine's $395.49 excess-shelter deduction, the formula benefit is $63.18 and the payable benefit is the $95 minimum."
-us,scenario_056,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used an excessive capped shelter deduction and obtained a $158 formula allotment. PolicyEngine allows $395.49 of excess shelter expense, leaving $805.03 net income and triggering the $95 minimum."
-us,scenario_056,snap,grok-4.3,llm_error,asset_resource,False,The model declared the household ineligible based on income and assets even though it passes both tests. It failed to apply the allowable deductions that reduce net income to $805.03 and establish the $95 monthly benefit.
-us,scenario_056,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,The model treated unlisted utility details as necessary and defaulted to zero despite the explicitly listed mortgage interest and real-estate taxes. PolicyEngine derives a $395.49 excess-shelter deduction from the supplied facts and pays the $95 monthly minimum.
-us,scenario_056,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied the earned-income and standard deductions but omitted the $395.49 excess-shelter deduction. It also concluded that a below-minimum formula amount becomes zero, whereas eligible one-person households receive the applicable $95 monthly minimum."
-us,scenario_056,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model imposed a $712 shelter cap instead of PolicyEngine's $395.49 excess-shelter deduction, reducing net income to about $493. PolicyEngine retains $805.03 of net income, so the formula amount falls below and is raised to the $95 minimum."
-us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output was missing."
-us,scenario_056,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $744 shelter cap and reduced net income to $466.25 instead of applying PolicyEngine's $395.49 excess-shelter deduction. The resulting PolicyEngine net income is $805.03, which makes the $95 monthly minimum controlling."
-us,scenario_056,snap,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly stated that income exceeds the one-person SNAP limits. Monthly gross income is $1,486.54 and monthly net income is $805.03, equal to 1.12 and 0.61 times the poverty guideline respectively, so both tests are passed."
-us,scenario_056,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly compared $17,839 with an annual gross limit it described as about $19,000 and called the income excessive despite it being lower. It then omitted the excess-shelter deduction and minimum-allotment rule that produce $95 per month."
-us,scenario_056,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model claimed both gross and net income exceed their limits because it stopped after the standard and earned-income deductions. Including the $395.49 excess-shelter deduction reduces net income to $805.03, while $1,486.54 gross income also passes its test."
+us,scenario_056,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model substituted an estimated $744 shelter cap for the $395.49 allowable excess-shelter deduction. That drove net income down to $462 and produced a formula allotment of $159 instead of $63.18, preventing application of the $95 minimum."
+us,scenario_056,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated the mortgage balance as a disqualifying asset and asserted that $1,486.54 of monthly gross income exceeded the applicable limit. The household passes the asset and income tests, and its shelter, standard, and earned-income deductions reduce net income to $805.03."
+us,scenario_056,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $395.49 excess-shelter deduction and therefore left monthly net income near $1,400. Including all $674.49 of allowable deductions produces $805.03 of net income and triggers the $95 monthly minimum allotment."
+us,scenario_056,snap,claude-opus-4.8,llm_error,asset_resource,False,The model inferred disqualifying assets from homeownership and financial assistance even though the household passes the resource test. It also failed to complete the deduction calculation that yields $805.03 of net income and a $95 monthly minimum benefit.
+us,scenario_056,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model compared total gross income directly with the SNAP limits and omitted the allowable standard, earned-income, and excess-shelter deductions. Those deductions total $674.49 monthly and reduce net income to $805.03, within the limit."
+us,scenario_056,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model stopped after the earned-income and standard deductions and used $1,216 as net income, omitting the $395.49 excess-shelter deduction. The resulting $805.03 net income gives a $63.18 formula benefit that is raised to the $95 minimum."
+us,scenario_056,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model incorrectly declared $1,486.54 of monthly gross income above the applicable SNAP threshold and treated financial assistance as an asset-like disqualifier. The household passes the gross, net, and resource tests."
+us,scenario_056,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model imposed an ABAWD disqualification by inferring insufficient work hours from annual self-employment income. The household passes the work-requirement test, so the benefit must be calculated from its eligible income and deductions."
+us,scenario_056,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $672 excess-shelter deduction instead of $395.49 and also used the wrong maximum allotment. The correct inputs produce $805.03 of net income, a $63.18 formula amount, and the controlling $95 monthly minimum."
+us,scenario_056,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly stated that $1,486.54 of monthly gross income exceeds the SNAP threshold. Gross income is only 1.12 times the $1,330 poverty guideline, and the household passes both income tests."
+us,scenario_056,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used a maximum excess-shelter deduction that reduced net income to about $489 rather than applying the trace's $395.49 shelter deduction. Correct net income is $805.03, so the formula amount falls below and is replaced by the $95 minimum."
+us,scenario_056,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model effectively reduced countable net income to zero and awarded the maximum allotment. The allowable deductions leave $805.03 of net income, so the maximum must be reduced by a $241.50 expected contribution before applying the $95 minimum."
+us,scenario_056,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount implies that the model overstated the shelter-related reduction or otherwise understated the $805.03 net income. The correct expected contribution is $241.50, leaving a sub-minimum formula benefit and a $95 monthly allotment."
+us,scenario_056,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,The model awarded the full maximum allotment merely because income and shelter deductions established eligibility. Eligibility does not eliminate the 30% net-income contribution: $805.03 of net income creates a $241.50 contribution and leaves only the $95 minimum.
+us,scenario_056,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,The model used an estimated $733 shelter cap as the deduction instead of the $395.49 allowable excess-shelter expense. This understated net income by roughly $338 and generated a $159.63 formula benefit rather than the controlling $95 minimum.
+us,scenario_056,snap,gpt-5.4-mini,llm_error,other,False,The model disregarded the prompt's complete-input convention and defaulted to zero instead of performing the specified calculation. The supplied income and shelter facts establish eligibility and yield the $95 monthly minimum allotment.
+us,scenario_056,snap,gpt-5.4-nano,llm_error,other,False,The model required unspecified additional low-income indicators even though the prompt supplies the complete facts and directs unlisted inputs to zero. Applying those facts yields $805.03 of net income and a positive minimum allotment.
+us,scenario_056,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly recognized that the formula benefit falls below the minimum but used an obsolete $23 monthly minimum. The applicable New Jersey minimum is $95 per month, producing $1,140 annually."
+us,scenario_056,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model left net income high enough to erase the allotment because it failed to apply the full $674.49 of allowable deductions. Correct net income is $805.03, and the resulting $63.18 formula benefit is raised to $95."
+us,scenario_056,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model overstated the capped excess-shelter deduction and reduced net income enough to calculate a $159 benefit. Only $395.49 of excess shelter expense is deductible here, leaving $805.03 of net income and invoking the $95 minimum."
+us,scenario_056,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used an excessive shelter deduction that produced a $158 formula allotment. The correct shelter deduction is $395.49, which leaves $805.03 of net income and makes the $95 minimum controlling."
+us,scenario_056,snap,grok-4.3,llm_error,asset_resource,False,The model asserted income and asset ineligibility without applying the household's passed resource test or its allowable deductions. The household passes every eligibility test and receives the minimum allotment.
+us,scenario_056,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,The model treated absent utility details as grounds for a zero benefit despite the prompt's instruction to set unlisted expenses to zero. The listed mortgage interest and real-estate taxes still generate a $395.49 excess-shelter deduction and a positive minimum benefit.
+us,scenario_056,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model looked only for rent and utility expenses and omitted the listed homeowner shelter expenses. Mortgage interest and real-estate taxes support a $395.49 excess-shelter deduction, reducing net income to $805.03."
+us,scenario_056,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,The model annualized the earned-income and standard deductions but omitted the excess-shelter deduction entirely. Applying the $395.49 monthly shelter deduction lowers net income enough to produce a positive formula amount and the $95 minimum.
+us,scenario_056,snap,inkling,llm_error,taxable_income_or_deductions,False,The model substituted a $712 shelter-cap estimate for the actual $395.49 excess-shelter deduction. That understated net income at about $493 rather than $805.03 and incorrectly kept the formula allotment above the minimum.
+us,scenario_056,snap,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable SNAP value or explanation. It therefore failed the required structured-output contract before any substantive SNAP calculation could be evaluated.
+us,scenario_056,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,The model deducted the full estimated $744 shelter cap rather than the $395.49 allowable excess-shelter amount. This reduced net income to $466.25 instead of $805.03 and produced a $158.125 formula benefit instead of the $95 minimum.
+us,scenario_056,snap,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly declared the household's income above the one-person SNAP limits. Gross income is 1.12 times the poverty guideline and net income after deductions is 0.61 times it, so both tests are passed."
+us,scenario_056,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model applied an excessive shelter-cap deduction and estimated net income near $500. The allowable $395.49 shelter deduction leaves $805.03 of net income, making the computed benefit $63.18 before the $95 minimum applies."
+us,scenario_056,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model incorrectly compared $17,839 with an approximate annual gross limit near $19,000 and declared it excessive despite its own figures. It also omitted the $395.49 excess-shelter deduction that reduces monthly net income to $805.03."
+us,scenario_056,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model omitted the excess-shelter deduction and consequently placed countable income above both gross and net limits. The household's $1,486.54 gross income passes the gross test, and the complete deductions reduce net income to $805.03."
 us,scenario_056,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_056,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,The model stopped at the 1.4% preliminary New Jersey tax calculation and reported $67.75. It failed to apply the New Jersey credits or adjustments that fully offset the $67.74 preliminary liability in the requested after-nonrefundable-credits output.
 us,scenario_056,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $12,000 of financial assistance in New Jersey gross income and deducted charitable contributions and the full property-tax amount to construct $3,462 of taxable income. The applicable computation uses only $1,498.53 of alimony and $4,340 of self-employment income, subtracts the $1,000 exemption, and then applies the adjustments that eliminate the resulting $67.74 preliminary tax."
 us,scenario_056,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for the requested output, so it failed the required output contract."
 us,scenario_056,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly treated the $12,000 financial assistance as New Jersey taxable income, invented a $3,066.18 self-employment-tax deduction against state income, and cycled through incompatible HSA and deduction treatments before asserting $299.43 without a supporting calculation. The correct state computation uses $5,838.53 of New Jersey AGI, subtracts the $1,000 exemption, calculates $67.74 of preliminary tax, and fully offsets that liability through applicable credits or adjustments."
-us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,The model incorrectly carried the federal childless-EITC minimum age of 25 into New Jersey's EITC rules. New Jersey's childless pathway covers the age-20 filer and produces a $265.60 refundable credit.
-us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that New Jersey has no state EITC or other refundable income-tax credit. The filer qualifies for New Jersey's childless EITC, which yields $265.60."
-us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,The model treated being under 21 and having no qualifying child as disqualifying. New Jersey's childless EITC pathway covers this 20-year-old and yields $265.60.
-us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly concluded that age 20, childlessness, and unearned income eliminate the New Jersey EITC. The filer meets New Jersey's childless age and income conditions, and $4,033.39 of earned income generates a $265.60 credit."
-us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model assumed New Jersey EITC eligibility requires satisfying the federal age-25 rule. It omitted New Jersey's childless EITC pathway for younger adults, under which this age-20 filer receives $265.60."
-us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model applied the federal childless-worker age range of 25–64 as New Jersey's controlling eligibility rule. New Jersey admits this age-20 filer through its childless EITC pathway and computes a $265.60 refundable credit.
-us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model imposed the federal age-25 childless-EITC restriction and also invented possible dependent status despite the prompt setting unlisted facts to false. The age-20 filer qualifies under New Jersey's childless EITC rules and receives $265.60.
-us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated a zero federal EITC as automatically producing a zero New Jersey EITC. It omitted New Jersey's childless eligibility pathway for this 20-year-old, which yields $265.60."
-us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model incorrectly made federal EITC qualification and the presence of children necessary for the New Jersey credit. New Jersey's childless pathway qualifies the age-20 filer and produces $265.60.
-us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits New Jersey's childless EITC pathway for younger adults. Applying that pathway to age 20 and $4,033.39 of earned income produces $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly recognized New Jersey's lower minimum age but incorrectly reduced the $309 phase-in amount to 40% and stopped at $123. PolicyEngine applies the 8% state phase-in directly, obtains $308.55 before reduction, and completes the calculation at $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly used New Jersey's age-18 childless pathway but incorrectly treated the state benefit as 40% of its $307 look-alike amount. The applicable computation uses the 8% phase-in on $4,033.39 and then the remaining reduction step, yielding $265.60."
-us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model incorrectly inferred that zero federal EITC and no qualifying children force the New Jersey credit to zero. New Jersey's childless EITC pathway independently covers this age-20 filer and yields $265.60.
-us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model used an erroneous age-21 cutoff and required dependents for New Jersey refundable-credit eligibility. The state's childless EITC pathway covers the 20-year-old and produces $265.60.
-us,scenario_056,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model propagated the federal childless-EITC age range of 25–64 into the New Jersey eligibility decision. New Jersey's younger childless-worker pathway qualifies this age-20 filer and yields $265.60.
-us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The zero answer fails to identify New Jersey's childless EITC pathway. The age-20 filer qualifies, and the earned-income calculation produces a $265.60 refundable credit."
-us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required additional explicit low-income indicators or children instead of applying the supplied age, AGI, and earned income. Those facts establish eligibility for New Jersey's childless EITC and generate $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the absence of dependents and positive state tax liability as eliminating refundable credits. The New Jersey childless EITC does not require a qualifying child or positive pre-credit liability, and it refunds $265.60 here."
-us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted the qualifying New Jersey childless EITC pathway. At age 20 with qualifying income and $4,033.39 of earned income, the filer receives $265.60."
-us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,other,False,The model substituted a $50 property-tax credit for the requested total refundable state credits and omitted the New Jersey EITC. The qualifying childless EITC alone contributes $265.60 to this output.
-us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,The model recognized the age-18 childless-worker rule but incorrectly calculated the credit as 40% of an unsupported $332 federal-formula amount. The state computation phases in at 8% to $308.55 before the final reduction to $265.60.
-us,scenario_056,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The zero answer omits New Jersey's childless EITC eligibility pathway for the age-20 filer. Applying it produces a refundable credit of $265.60.
-us,scenario_056,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly made a positive federal EITC a prerequisite for any New Jersey EITC. New Jersey's childless pathway covers this younger filer and yields $265.60 despite the federal age rule.
-us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated the New Jersey EITC solely as 40% of the actual federal EITC and therefore stopped at zero. It omitted New Jersey's childless pathway for the age-20 filer, which produces $265.60; the property-tax deduction does not remove that credit."
-us,scenario_056,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model incorrectly equated federal EITC ineligibility with New Jersey EITC ineligibility. New Jersey's childless rules cover this age-20 filer without dependents and yield $265.60.
-us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits. It therefore failed the required output contract before any substantive credit calculation could be evaluated.
-us,scenario_056,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,The model applied the federal 25–64 childless-worker age range to New Jersey and set the state EITC to zero. New Jersey's younger childless-worker pathway qualifies this 20-year-old and yields $265.60; the nonrefundable property-tax credit is irrelevant to that result.
-us,scenario_056,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The zero answer overlooks New Jersey's childless EITC pathway for younger workers. The age-20 filer qualifies based on age and income, producing $265.60."
-us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of children and the earned-income profile as disqualifying. New Jersey's childless EITC expressly covers this age-20 filer, and $4,033.39 of earned income leads to a $265.60 credit."
-us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly required qualifying children, positive New Jersey tax liability, or other unstated conditions for a refundable credit. The childless New Jersey EITC is refundable without positive pre-credit liability and equals $265.60 here."
+us,scenario_056,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,The model applied the federal minimum age of 25 directly to the New Jersey EITC. New Jersey's childless EITC pathway covers this age-20 filer and yields $265.60 after phase-in and reduction.
+us,scenario_056,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that New Jersey has no state EITC or other refundable income-tax credits. The New Jersey childless EITC applies to this 20-year-old and produces $265.60.
+us,scenario_056,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated being age 20 without a qualifying child as disqualifying for the New Jersey EITC. The state childless-worker age rule covers this filer, so the earned-income calculation produces a refundable credit."
+us,scenario_056,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model wrongly concluded that age 20, childlessness, and unearned income eliminate the New Jersey EITC. The filer qualifies through New Jersey's childless EITC pathway, with $4,033.39 of earned income generating the credit."
+us,scenario_056,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model made federal childless EITC age eligibility a prerequisite for the New Jersey credit. New Jersey's expanded age pathway admits this 20-year-old and yields $265.60.
+us,scenario_056,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model imported the federal age-25 minimum into New Jersey's childless EITC rules. New Jersey independently covers this age-20 filer, so the absence of children does not reduce the state credit to zero."
+us,scenario_056,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model applied the federal age-25 childless-worker restriction and invented a possible dependent status despite the prompt setting unlisted statuses to false. New Jersey's childless EITC covers the age-20 filer and produces $265.60.
+us,scenario_056,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model assumed a zero federal EITC necessarily makes the New Jersey EITC zero. It omitted New Jersey's childless EITC eligibility pathway for younger workers, which covers this 20-year-old."
+us,scenario_056,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model incorrectly conditioned the New Jersey EITC on federal EITC eligibility. The state-specific childless-worker age rule qualifies this age-20 filer despite federal age ineligibility.
+us,scenario_056,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits New Jersey's childless EITC pathway for an age-20 worker. Applying that pathway to $4,033.39 of earned income and the required reduction yields $265.60."
+us,scenario_056,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model correctly recognized New Jersey's expanded age eligibility but incorrectly capped the credit at 40% of the $308.55 phase-in amount. PolicyEngine applies the New Jersey computation and subsequent reduction directly, producing $265.60 rather than $123."
+us,scenario_056,state_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model correctly used the younger-worker eligibility pathway but incorrectly multiplied a federal look-alike amount by 40%. The applicable computation phases in the credit at 8% using $4,033.39 of earned income and then reduces it to $265.60."
+us,scenario_056,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model treated zero federal EITC and no qualifying children as dispositive. New Jersey's childless EITC has a state-specific younger-worker pathway that qualifies this 20-year-old.
+us,scenario_056,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model used an incorrect age-21 cutoff for New Jersey refundable-credit eligibility. The New Jersey childless EITC covers this age-20 filer and does not require a dependent.
+us,scenario_056,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,The model carried the federal childless EITC age range of 25–64 into the New Jersey determination. New Jersey's expanded childless-worker rule qualifies the 20-year-old independently.
+us,scenario_056,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The zero answer overlooks the New Jersey childless EITC available to this age-20 worker. The state pathway and earned-income computation yield $265.60.
+us,scenario_056,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required children or additional explicit low-income indicators. The stated age, state, adjusted gross income, and earned income establish eligibility for New Jersey's childless EITC."
+us,scenario_056,state_refundable_credits,gpt-5.5,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of dependents and positive state tax liability as barring refundable credits. The New Jersey childless EITC requires neither a qualifying child nor positive pre-credit liability and yields $265.60.
+us,scenario_056,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted New Jersey's childless EITC eligibility pathway for younger workers. This age-20 filer qualifies, and the applicable earned-income calculation produces $265.60."
+us,scenario_056,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model substituted New Jersey's $50 property-tax credit for the refundable-credit output and omitted the New Jersey EITC. The requested refundable total is driven entirely by the $265.60 childless EITC, not the property-tax alternative."
+us,scenario_056,state_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model recognized the age-18 childless-worker pathway but used an incorrect $332 base and then multiplied it by 40%. The trace uses $4,033.39 of earned income at an 8% phase-in rate to reach $308.55, followed by the required reduction to $265.60."
+us,scenario_056,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The zero answer omits New Jersey's childless EITC for younger workers. The age-20 filer meets that state eligibility pathway and receives $265.60.
+us,scenario_056,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model assumed that zero federal EITC forces the New Jersey EITC to zero. New Jersey's state-specific childless-worker age expansion qualifies this 20-year-old.
+us,scenario_056,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly made receipt of federal EITC necessary for the New Jersey credit. The younger childless-worker pathway applies independently and yields $265.60.
+us,scenario_056,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model reduced the New Jersey EITC to 40% of an actually received federal EITC and therefore treated federal age ineligibility as controlling. It missed New Jersey's childless EITC pathway for this age-20 worker.
+us,scenario_056,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model treated federal EITC ineligibility and the absence of dependents as eliminating the New Jersey EITC. New Jersey's expanded childless-worker age rule covers the filer and produces $265.60.
+us,scenario_056,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits. It therefore failed the required output contract.
+us,scenario_056,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,The model applied the federal age-25 minimum to the New Jersey EITC and treated the state credit solely as 40% of an actually received federal credit. New Jersey's younger childless-worker pathway qualifies this age-20 filer.
+us,scenario_056,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The zero answer omits New Jersey's childless EITC pathway for an age-20 worker. The household composition does not bar that credit, which equals $265.60 after the earned-income calculation and reduction."
+us,scenario_056,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,The model treated federal age ineligibility as eliminating the New Jersey EITC. New Jersey's childless-worker age expansion qualifies this 20-year-old even without a child.
+us,scenario_056,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or different earned-income characteristics for New Jersey EITC eligibility. This childless 20-year-old qualifies, and $4,033.39 of earned income generates the credit."
+us,scenario_056,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly treated the absence of children and state tax liability as disqualifying and overlooked the eligible earned income. New Jersey's refundable childless EITC applies regardless of positive pre-credit liability and yields $265.60.
 us,scenario_056,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly identified the maximum-credit region but substituted an estimated $659 maximum and then applied an unsupported lookup-table reduction to $636. The applicable 2026 childless EITC maximum is $664, with no such reduction."
-us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the standard deduction and zero taxable income as eliminating refundable credits. EITC is refundable and is computed from earned self-employment income and AGI independently of positive federal income-tax liability; $10,400 is also below, not above, the relevant joint-filer limit."
-us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The reasoning establishes that the head satisfies the age test and that income lies on the maximum-credit plateau, but the submitted $397 contradicts its own calculated maximum-credit result. Applying the 2026 maximum yields $664."
-us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly placed roughly $9,600 of earned income in the childless EITC phaseout region. For married filing jointly, phaseout begins at a substantially higher income, so this household receives the full 2026 maximum of $664."
-us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model acknowledged that joint-filer phaseout begins around $18,000 but nevertheless reduced the credit at $10,400. This income remains in the maximum-credit region, where the 2026 childless EITC is $664."
-us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model alternately described a maximum near $649 and an estimate near $600, then submitted $967, which exceeds the applicable childless EITC maximum. The spouses do not receive separate worker credits; the joint return receives one $664 EITC."
-us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,The model correctly placed the household on the childless EITC plateau but reused an estimated 2025 maximum of $649. The applicable 2026 maximum is $664.
-us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model estimated the 2026 maximum by informally inflating the 2024 amount to $630 instead of applying the actual 2026 parameter. The plateau credit is $664.
-us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or rejected the household under the earned-income limits. Childless joint filers qualify when the age and income rules are met, and the head's self-employment income produces the $664 EITC."
-us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly identified the maximum-credit plateau but used $666 instead of the applicable 2026 childless EITC maximum. The parameter yields $664.
-us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model recognized childless EITC eligibility but used an unsupported estimated 2026 amount of $620. The household is on the plateau and receives the statutory 2026 maximum of $664.
-us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model identified the correct credit and eligibility pathway but rounded or substituted the maximum as $660. The applicable 2026 childless EITC amount is exactly $664.
-us,scenario_057,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model correctly recognized that self-employment supports a childless EITC but applied a $649 maximum from the wrong parameter year. The 2026 maximum is $664.
-us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a fictitious $1,231–$1,283 childless EITC maximum and an internally inconsistent phase-in calculation. Under the actual 2026 no-child schedule, the maximum refundable credit is $664."
-us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the absence of wages and qualifying children as precluding EITC. Net earnings from self-employment are earned income, and a qualifying child is not required for the childless EITC, producing $664 here."
-us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The zero answer omits the childless EITC generated by the head's self-employment earnings. The household satisfies the no-child joint-filer rules and receives $664.
-us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model correctly identified eligibility and the maximum-credit region but used an estimated maximum of $667. The applicable 2026 maximum is $664.
-us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly concluded that the household receives the maximum childless-worker EITC but used $649, a parameter from the wrong year. The 2026 maximum is $664."
-us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model incorrectly treated the spouse's age of 20 as disqualifying the joint return. For a married joint return without qualifying children, the age condition is satisfied when one spouse qualifies; the 43-year-old head does, yielding a $664 EITC."
-us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly concluded that income phaseout reduces the EITC to zero. At this income, a childless married couple remains well below the joint phaseout threshold and receives the $664 maximum."
-us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied the age test, self-employment earned-income treatment, and plateau logic but used a projected $667 maximum. The actual 2026 maximum is $664."
-us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,The model correctly placed the household below phaseout and at the maximum but used $648 from an obsolete or estimated parameter set. The 2026 childless EITC maximum is $664.
-us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"The model applied the minimum-age rule solely to the 20-year-old spouse and ignored the qualifying 43-year-old head. One spouse satisfying the age condition is enough on this joint return, so the household receives $664."
-us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"The model incorrectly required both spouses on a joint return to be at least 25. The 43-year-old head satisfies the childless EITC age rule for the couple, and their earned income produces $664."
-us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly treated $10,400 of self-employment income as exceeding the childless joint-filer EITC limit. It lies in the maximum-credit region, producing a $664 EITC."
-us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly said low income and disability benefits defeat EITC eligibility. Self-employment earnings count as earned income, and the small disability-benefit amounts do not erase the resulting $664 childless EITC."
-us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model ignored self-employment earnings because there were no wages and incorrectly treated the absence of children as barring EITC. Net self-employment earnings are earned income, and the no-child EITC pathway yields $664."
+us,scenario_057,federal_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model correctly placed the household on the childless EITC plateau but replaced the 2026 maximum of $664 with an unsupported $636 lookup-table estimate. Lookup-table rounding does not reduce the plateau credit to $636.
+us,scenario_057,federal_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated zero taxable income after the standard deduction as eliminating refundable credits. EITC is refundable and is computed from earned income and AGI, so the self-employment earnings generate the $664 childless EITC despite no pre-credit income-tax liability."
+us,scenario_057,federal_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model’s reasoning correctly establishes age eligibility and placement on the maximum-credit plateau, but its submitted $397 contradicts its own plateau calculation. The applicable 2026 childless EITC maximum is $664, not $397 or the stated $649 estimate."
+us,scenario_057,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly placed roughly $9,600 of net self-employment earnings in the married-joint childless EITC phaseout region. That income remains below the joint phaseout threshold and lies on the maximum-credit plateau, yielding $664."
+us,scenario_057,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly applied a plateau or phaseout adjustment at $10,400 of self-employment income. The household is below the married-joint phaseout threshold and receives the full 2026 childless EITC maximum of $664."
+us,scenario_057,federal_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model cited the 7.65% childless EITC structure and a maximum near $649 but submitted $967, an amount incompatible with both that rate and the statutory maximum. The couple receives one household-level childless EITC capped at $664; it is not increased by treating both spouses as workers."
+us,scenario_057,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,The model correctly identified the maximum-credit plateau but carried forward an estimated 2025-based maximum of $649. The applicable 2026 childless EITC maximum is $664.
+us,scenario_057,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly identified plateau eligibility but estimated the 2026 maximum by informally inflating the 2024 amount to $630. The actual 2026 parameter produces a maximum childless EITC of $664.
+us,scenario_057,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly denied EITC eligibility based on the absence of a qualifying child or an unspecified earned-income threshold. Childless married filers qualify when the age and income rules are met, and this household’s self-employment earnings generate the $664 maximum credit."
+us,scenario_057,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly placed the household on the maximum childless EITC plateau but used $666 instead of the applicable 2026 maximum of $664.
+us,scenario_057,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,The model recognized childless EITC eligibility but substituted an unsupported $620 estimate for the 2026 plateau amount. The applicable maximum is $664.
+us,scenario_057,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model correctly identified the qualifying childless EITC but used $660 instead of the exact 2026 maximum of $664.
+us,scenario_057,federal_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model correctly recognized that self-employment income qualifies for the childless EITC but used an outdated $649 maximum. The 2026 maximum is $664.
+us,scenario_057,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a nonexistent $1,231–$1,283 maximum for the ordinary 2026 childless EITC and also derived an unsupported $8,320 earned-income figure. Under the actual 7.65% schedule, the household is capped at the $664 maximum."
+us,scenario_057,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of wages and qualifying children as eliminating EITC. Net earnings from self-employment are earned income, and a qualifying child is not required for the childless EITC, which equals $664 here."
+us,scenario_057,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to apply the childless EITC pathway to the head’s self-employment earnings. Those earnings trigger the maximum 2026 childless credit of $664.
+us,scenario_057,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly determined that the head satisfies the age test and that income is on the maximum-credit plateau, but used $667 as the projected maximum. The applicable 2026 maximum is $664."
+us,scenario_057,federal_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,The model correctly found maximum childless-worker EITC eligibility but used the outdated $649 maximum rather than the 2026 maximum of $664.
+us,scenario_057,federal_refundable_credits,gpt-5.6-terra,llm_error,age_disability,False,"The model incorrectly required the 20-year-old spouse to satisfy the childless EITC age test. On a joint return, eligibility is satisfied when either spouse meets the age requirement; the 43-year-old head does, producing a $664 EITC."
+us,scenario_057,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,The model incorrectly concluded that the household’s income phases the childless EITC down to zero. The self-employment income is below the married-joint phaseout threshold and places the household on the $664 maximum-credit plateau.
+us,scenario_057,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied the age rule, net self-employment earnings, and plateau treatment but used a projected $667 maximum. The exact 2026 childless EITC maximum is $664."
+us,scenario_057,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,The model correctly placed earned income and AGI between the plateau start and joint phaseout start but used $648 as the 2026 maximum. The applicable maximum is $664.
+us,scenario_057,federal_refundable_credits,inkling,llm_error,age_disability,False,"The model incorrectly applied the age-25 minimum separately to the 20-year-old spouse and denied the joint return. The 43-year-old head satisfies the childless EITC age requirement for the couple, yielding $664."
+us,scenario_057,federal_refundable_credits,kimi-k2.6,llm_error,age_disability,False,"The model incorrectly required both spouses on a joint return to be at least 25. The childless EITC age condition is met because the head is 43, so the qualifying earnings produce the $664 maximum credit."
+us,scenario_057,federal_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model incorrectly treated $10,400 of SSTB self-employment income as exceeding the childless EITC limit. SSTB status does not impose a separate EITC cutoff, and this income is below the married-joint phaseout threshold, producing $664."
+us,scenario_057,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,The model correctly located the household on the full-credit plateau but estimated the 2026 maximum by indexing an earlier amount to $625. The actual 2026 maximum is $664.
+us,scenario_057,federal_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly asserted that low income and disability benefits defeat EITC eligibility. The self-employment income is qualifying earned income, and the small disability-benefit amounts do not eliminate the childless EITC; the result is $664."
+us,scenario_057,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly equated the absence of wage income and qualifying children with no EITC. Self-employment earnings count as earned income, and the childless EITC pathway produces a $664 credit."
 us,scenario_057,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model looked only to the disability flag and disability-benefit amount and failed to account for the head's SSI receipt. SSI receipt places the head in Louisiana's automatically eligible SSI_RECIPIENT Medicaid category.
 us,scenario_057,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model restricted Medicaid eligibility to expansion, pregnancy, parent/caretaker, and aged pathways and omitted automatic SSI-recipient eligibility. The head's $7,231.00146484375 in SSI establishes eligibility through the SSI_RECIPIENT category regardless of those demographic categories or the MAGI test."
 us,scenario_057,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated disability alone as sufficient for Medicare regardless of age. A disability flag does not establish the required SSDI entitlement and 24-month waiting period, and the 43-year-old head does not meet the age-65 rule."
@@ -3636,36 +3820,38 @@ us,scenario_057,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out
 us,scenario_057,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model correctly computed the ordinary self-employment tax and then wrongly imposed the 0.9% Additional Medicare Tax above $400. The $400 figure is the net-earnings threshold for self-employment-tax filing, while the Additional Medicare threshold is $250,000 for joint filers and that tax is expressly excluded from this output."
 us,scenario_057,self_employment_tax,qwen-3.7-max,llm_error,thresholds_rates,False,"The model invented a $12,000 exemption threshold for self-employment tax. The applicable net-earnings threshold is $400, so $10,400 of self-employment income generates $1,469.47 of tax after the 92.35% adjustment."
 us,scenario_057,self_employment_tax,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model correctly calculated $1,469.47 and then reported approximately one-half of it. One-half of self-employment tax is deductible in computing adjusted gross income; it does not reduce the self-employment-tax liability reported by this output."
-us,scenario_057,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $602.58 monthly SSI receipt from SNAP countable income and used only self-employment and the listed disability benefits. That reduced monthly net income to about $485 instead of $1,090.08 and materially overstated the allotment."
-us,scenario_057,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model acknowledged that the household had low income and assets but then assigned zero without applying the eligibility tests or benefit formula. The household passes the net, gross, asset, categorical-eligibility, and work tests and therefore receives a positive allotment."
-us,scenario_057,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the engine-computed SSI receipt from countable income and then submitted $6,240 despite its own intermediate benefit calculation of about $3,990. The correct monthly net income is $1,090.08, not the low annual amount used in its calculation."
-us,scenario_057,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted $602.58 per month of SSI and inconsistently converted the $38 annual disability-benefit input into $456 of unearned income. It consequently calculated net income far below $1,090.08 per month and overstated SNAP."
-us,scenario_057,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model treated net income as nearly zero and used an incorrect two-person maximum of about $304 per month. The applicable maximum is $546 for most months and $558.24 later, reduced by a 30% contribution based on $1,090.08 of monthly net income."
-us,scenario_057,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a speculative 40% self-employment business-expense deduction while omitting the $602.58 monthly SSI benefit from countable income. Those choices drove its monthly net income down to $205.17 instead of $1,090.08 and produced a near-maximum allotment."
-us,scenario_057,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied an unsupported 50% self-employment reduction and mishandled annual versus monthly disability and medical amounts. It also failed to incorporate the engine-computed $602.58 monthly SSI receipt, so its stated income derivation cannot support the submitted benefit."
-us,scenario_057,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model counted only self-employment and $38 of disability benefits, omitting $602.58 per month of SSI. This produced net income of $489.34 rather than $1,090.08 and overstated the monthly benefit."
-us,scenario_057,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model inserted $13,122.96 of SSI instead of using the engine-computed $602.58 monthly SSI amount and also invented a $60 excess-medical deduction. Its resulting annual net income and expected contribution were therefore too high, sharply understating SNAP."
-us,scenario_057,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer implies that the model failed to apply the household's passing net-income, gross-income, asset, categorical-eligibility, and work tests. With monthly net income of $1,090.08, the benefit formula yields a positive allotment."
-us,scenario_057,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model reduced the benefit to about $76 per month by using an incorrect SSI-inclusive countable-income figure. Correct deductions produce monthly net income of $1,090.08 and benefits of $219 or $232.74, depending on the month."
-us,scenario_057,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model inserted $12,729 of SSI rather than the engine-computed $602.58 monthly receipt. That overstated countable income and the expected household contribution, reducing SNAP to $776."
-us,scenario_057,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $268 monthly average does not apply the traced monthly net income of $1,090.08 to the applicable maximum allotments. The correct month-specific results are $219 and $232.74, totaling $2,669.22."
-us,scenario_057,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model recognized eligibility but used an unsupported $282 monthly allotment. Applying the 30% contribution to $1,090.08 of monthly net income yields $219 in most months and $232.74 later."
-us,scenario_057,snap,glm-5.2,llm_error,categorical_eligibility,False,The model misread the instruction that unlisted inputs equal zero as requiring an unlisted benefit output to equal zero. It needed to estimate SNAP from the supplied household facts and assumed take-up; the household qualifies and receives a positive computed benefit.
-us,scenario_057,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly declared countable income too high. Monthly gross income of $1,472.42 and net income of $1,090.08 are only 84% and 62% of their respective poverty-guideline limits, so the household passes both income tests."
-us,scenario_057,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of a listed SNAP receipt as a zero benefit instead of calculating eligibility and assuming take-up as instructed. The household passes all eligibility tests and the allotment formula produces a positive annual amount.
-us,scenario_057,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted the $602.58 monthly SSI receipt and based net income only on self-employment and listed disability benefits. Its annual net income of $5,862 is therefore far below the traced $1,090.08 monthly net income, overstating SNAP."
-us,scenario_057,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $340 monthly estimate fails to include the full countable-income result, particularly the engine-computed SSI receipt. With monthly net income of $1,090.08, the expected contribution is about $327 and the benefit is $219 in most months."
-us,scenario_057,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model derived a $476 expected contribution and $70 benefit from an overstated countable-net-income figure. Correct deductions leave $1,090.08 of monthly net income, producing an expected contribution near $327 rather than $476."
-us,scenario_057,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overcounted SSI, disability, or self-employment income after deductions, producing only a $70 monthly allotment. The correct monthly net income is $1,090.08, which yields $219 in most months and $232.74 later."
-us,scenario_057,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly asserted that income or assets exceed the limits. Bank assets are only $911.76, and gross and net income equal 84% and 62% of their respective poverty-guideline thresholds, so all relevant tests are passed."
-us,scenario_057,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used about $1,928 of monthly gross income by adding an excessive SSI amount. The traced gross income is $1,472.42 and net income is $1,090.08, so its expected contribution was overstated."
-us,scenario_057,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted only $209 for the standard deduction from an annual income figure, mixing monthly and annual periods, and omitted SSI from gross income. The engine computes income and deductions monthly before annualizing the resulting allotments."
-us,scenario_057,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model omitted $602.58 per month of SSI and therefore used monthly net income of about $488 rather than $1,090.08. That omission reduced the expected contribution and overstated the allotment to roughly $399 per month."
-us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, so it failed the required output contract."
-us,scenario_057,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model inserted $13,183 of estimated SSI, producing monthly net income of $1,586.08. The applicable engine-computed SSI receipt is $602.58 per month and correct deductions produce net income of $1,090.08."
-us,scenario_057,snap,minimax-m3,llm_error,asset_resource,False,"The model reversed the asset comparison: $912 is below, not above, an asset limit of about $4,500. The traced bank assets are $911.76, so the household passes the asset test."
-us,scenario_057,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded all income merely because household members receive or qualify for SSI. SSI and applicable self-employment income remain part of the SNAP income calculation, which yields $1,090.08 of monthly net income rather than zero; it also used an inflated $635 maximum allotment."
-us,scenario_057,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly denied a positive benefit without applying the traced tests. The household passes the net-income, gross-income, asset, categorical-eligibility, and work-requirement tests, after which the allotment formula yields a positive amount."
+us,scenario_057,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the household's $602.58 monthly SSI from SNAP countable income and calculated from only self-employment and the listed disability benefits. Including SSI raises monthly net income to $1,090.08 and reduces the annual allotment to $2,669.22."
+us,scenario_057,snap,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced self-employment income by the 92.35% self-employment-tax factor and then asserted a zero benefit despite acknowledging very low income and assets. SNAP uses its own earned-income deduction, and the resulting eligible household receives $2,669.22."
+us,scenario_057,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted $602.58 per month of SSI from countable income and then submitted $6,240 despite its own intermediate calculation producing about $3,990. The correct monthly net income is $1,090.08, producing $2,669.22 annually."
+us,scenario_057,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the generated SSI benefit from countable income, misread the $14 and $24 annual disability amounts as $456 of annual unearned income, and submitted a figure inconsistent with its own $4,536 calculation. Countable gross income is $1,472.42 monthly and the annual SNAP benefit is $2,669.22."
+us,scenario_057,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model used an erroneous two-person maximum allotment of about $304 per month and treated net income as nearly zero. The applicable maxima are $546 and later $558.24, while monthly net income is $1,090.08 after counting SSI and deductions."
+us,scenario_057,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted $602.58 per month of SSI and invented a 40% self-employment business-expense deduction not present in the traced calculation. Those errors reduced its net income to $205.17 monthly instead of $1,090.08 and overstated SNAP."
+us,scenario_057,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model applied unrelated self-employment-tax and 50% expense reductions, treated annual disability and medical amounts as monthly, and never correctly incorporated $602.58 monthly SSI. SNAP net income is $1,090.08 per month, not the internally contradictory figures in its explanation."
+us,scenario_057,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model calculated monthly gross income as only $870 and omitted the $602.58 monthly SSI benefit. This understated net income from $1,090.08 to $489.34 and consequently overstated the allotment."
+us,scenario_057,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model counted $13,122.96 of SSI instead of the traced $7,230.96 annual SSI and also invented a $60 medical deduction despite only $40 of annual expenses. The correct monthly gross and net incomes are $1,472.42 and $1,090.08."
+us,scenario_057,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly declared the household ineligible without applying the traced eligibility tests, including categorical eligibility through SSI and the net-income test. The eligible household receives $2,669.22 annually."
+us,scenario_057,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $76 monthly allotment implies excessive countable income after SSI and deductions. Using $602.58 monthly SSI produces $1,090.08 of net income and benefits of $219 in most months, rising to $232.74 later."
+us,scenario_057,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model inflated unearned income to $12,729 instead of using $7,230.96 of annual SSI plus $38 of disability benefits. That excessive income understated the SNAP allotment."
+us,scenario_057,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model recognized eligibility but its $268 monthly shortcut did not apply the traced maximum allotments and 30% contribution to $1,090.08 of monthly net income. The month-specific benefits are $219 and later $232.74."
+us,scenario_057,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,The model recognized eligibility but its $282 monthly estimate did not subtract the correct 30% contribution from the applicable two-person maxima. The traced computation yields $219 in most months and $232.74 in later months.
+us,scenario_057,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model treated the instruction that unlisted inputs equal zero as an instruction to set the requested computed benefit to zero. SNAP had to be calculated from the facts, and the household passes every eligibility test and receives $2,669.22."
+us,scenario_057,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly declared counted income too high. Monthly gross income is only 84% of the poverty guideline, monthly net income is 62%, and SSI also establishes categorical eligibility."
+us,scenario_057,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated absence of a listed SNAP receipt amount as zero instead of computing eligibility and benefits under the required take-up assumption. The household qualifies and receives $2,669.22."
+us,scenario_057,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted $7,230.96 of annual SSI from SNAP countable income, using only $10,438 of self-employment and disability income. This understated net income and overstated the benefit by more than $2,100."
+us,scenario_057,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $340 monthly estimate understates the household's contribution because it does not fully count the $602.58 monthly SSI benefit. With net income of $1,090.08, the benefit is $219 in most months and $232.74 later."
+us,scenario_057,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $70 monthly result implies net income of roughly $1,587, rather than the traced $1,090.08 after the earned-income and standard deductions. Applying the correct net income to the $546 maximum gives $219 in the ordinary months."
+us,scenario_057,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model counted SSI but arrived at excessive net income consistent with inflating the SSI amount or failing to apply the deductions correctly. Monthly net income is $1,090.08, so the ordinary monthly allotment is $219 rather than $70."
+us,scenario_057,snap,grok-4.3,llm_error,asset_resource,False,"The model incorrectly asserted that income or assets exceed SNAP limits. Bank assets are only $911.76, gross and net income are 84% and 62% of their respective guidelines, and the household is categorically eligible through SSI."
+us,scenario_057,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model inflated monthly SSI-inclusive gross income to about $1,928 instead of $1,472.42. After the earned-income and standard deductions, net income is $1,090.08 rather than $1,551."
+us,scenario_057,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model inflated monthly gross income to about $1,955 and net income to $1,578. The traced SSI-inclusive figures are $1,472.42 gross and $1,090.08 net, yielding a substantially larger allotment."
+us,scenario_057,snap,grok-build-0.1,llm_error,period_annualization,False,"The model subtracted a $209 monthly standard deduction only once from annual income instead of applying it across the year, and it omitted annual SSI income. SNAP is calculated monthly with changing parameters, producing $2,669.22 across the year."
+us,scenario_057,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model omitted $602.58 of monthly SSI and based the benefit on only about $870 of gross income. Correct gross income is $1,472.42 monthly and net income after deductions is $1,090.08."
+us,scenario_057,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SNAP value or explanation, so the required output was missing."
+us,scenario_057,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model estimated $13,183 of annual SSI instead of the traced $7,230.96, inflating monthly net income to $1,586.08. Correct net income is $1,090.08 and produces $2,669.22 annually."
+us,scenario_057,snap,minimax-m3,llm_error,asset_resource,False,"The model reversed the asset comparison: $912 is below, not above, the roughly $4,500 disabled-household limit it cited. The traced asset amount is $911.76, so the household passes the asset test."
+us,scenario_057,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model omitted the household's $602.58 monthly SSI and calculated gross income as only about $870. Including SSI raises net income to $1,090.08 and lowers the annual benefit to $2,669.22."
+us,scenario_057,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,The model incorrectly excluded all income merely because both adults are disabled and SSI-eligible; SSI and the household's earned income remain countable in the SNAP benefit calculation. It also used an inflated $635 maximum instead of the traced $546 and $558.24 monthly maxima.
+us,scenario_057,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly denied a positive benefit without applying the traced tests. The household passes the gross-income, net-income, asset, categorical-eligibility, and work-requirement tests and receives $2,669.22."
 us,scenario_057,spouse_chip_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated Louisiana CHIP as covering low-income adults through a standalone income test. It failed to apply the controlling Medicaid exclusion: the 20-year-old spouse is Medicaid-eligible in the adult category and therefore is not eligible for CHIP, regardless of being below an asserted CHIP income threshold."
 us,scenario_057,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model failed to apply Louisiana's ACA adult expansion pathway and incorrectly demanded additional qualifying details. At age 20 with MAGI of 0.45 times FPL, the spouse qualifies in the MAGI-based ADULT category without another categorical condition."
 us,scenario_057,spouse_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated pregnancy or parenthood of a minor child as necessary for Louisiana Medicaid expansion. Louisiana's ACA expansion covers low-income adults under 65 regardless of pregnancy or parental status, and the spouse's MAGI of 0.45 times FPL satisfies its financial test."
@@ -3676,64 +3862,67 @@ us,scenario_057,spouse_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,Fa
 us,scenario_057,spouse_medicare_eligible,kimi-k3,llm_error,age_disability,False,The model invented a PolicyEngine rule under which the disability boolean itself satisfies under-65 Medicare eligibility. The spouse is below 65 and has no listed qualifying Medicare entitlement pathway.
 us,scenario_057,spouse_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented WIC coverage for women up to age 60 and treated disability as a route to postpartum or other categorical eligibility. The spouse is not pregnant, breastfeeding, or postpartum, so satisfying an income limit cannot establish WIC eligibility."
 us,scenario_057,spouse_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated being a 20-year-old disabled woman as a qualifying WIC category. Age, sex, and disability do not replace the required pregnancy, breastfeeding, or postpartum status, none of which is present."
-us,scenario_057,ssi,claude-fable-5,llm_error,missing_output,False,"The model submitted a placeholder zero without applying SSI eligibility or benefit rules. The head passes the disability and resource tests, and the individual calculation after spousal deeming yields $7,231."
-us,scenario_057,ssi,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated nearly all $10,400 of self-employment income as countable after only a $65-per-month exclusion and therefore wrongly eliminated the benefit. PolicyEngine's earned-income exclusions and spousal-deeming calculation produce $391.42 in monthly countable income, leaving $602.58 per month for the eligible head."
-us,scenario_057,ssi,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the spouse as SSI-disabled and calculated one pooled eligible-couple benefit. SSI is individual: the spouse fails the aged/blind/disabled criterion, while the head receives an individual benefit with spousal deeming, totaling $7,231."
-us,scenario_057,ssi,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly declared both spouses eligible and started from the couple federal benefit rate. Only the head satisfies the SSI aged/blind/disabled test, so the individual rate and spousal-deemed income produce $7,231."
-us,scenario_057,ssi,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the self-employment income as exceeding the applicable SSI maximum without correctly applying the earned-income disregards. The resulting monthly countable income is $391.42, below the eligible head's individual maximum, leaving $7,231 annually."
-us,scenario_057,ssi,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,The model assumed the spouse's generic disability flag made both people SSI-eligible and used a couple benefit rate. The spouse fails PolicyEngine's SSI aged/blind/disabled criterion; only the head receives the individual benefit after spousal deeming.
-us,scenario_057,ssi,claude-sonnet-5,llm_error,other,False,"The reasoning itself derived a positive SSI payment but the model submitted zero, contradicting its own calculation. It also used an eligible-couple framework instead of calculating the eligible head individually with spousal deeming."
-us,scenario_057,ssi,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model subtracted countable income from an estimated eligible-couple maximum. The spouse is not SSI-eligible, so the head must be calculated under the individual maximum with spousal deeming, yielding $7,231."
-us,scenario_057,ssi,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model treated both spouses as SSI-eligible and used the couple federal benefit rate. Only the head meets the SSI categorical test, and the head's individual computation with deemed spousal income yields $7,231."
-us,scenario_057,ssi,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated $10,400 of earned income as disqualifying without applying the SSI earned-income exclusions. Those rules reduce monthly countable income to $391.42, leaving a positive $7,231 annual payment for the head."
-us,scenario_057,ssi,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model explicitly used a couple maximum, incorrectly treating the spouse as categorically eligible. SSI is calculated for the eligible head as an individual with spousal deeming, not as a jointly eligible couple."
-us,scenario_057,ssi,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly classified both adults as an eligible SSI couple and subtracted income from a couple maximum. The spouse fails the SSI aged/blind/disabled criterion, while the head's individual benefit equals $7,231."
-us,scenario_057,ssi,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model treated the two adults as an SSI-eligible couple solely from their disability descriptions. Only the head meets PolicyEngine's SSI categorical criterion, so the correct unit is the head individually with spousal deeming."
-us,scenario_057,ssi,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly stated that both spouses form an eligible SSI couple. The spouse is not SSI aged, blind, or disabled under the operative criterion, and only the head receives $7,231 after deemed income."
-us,scenario_057,ssi,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model treated $10,400 of self-employment income as exceeding SSI limits and also relied on SSI receipt not being listed, despite assumed take-up. After SSI earned-income exclusions, the eligible head has $391.42 of monthly countable income and receives $7,231."
-us,scenario_057,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model falsely stated that $912 of assets exceeds the SSI asset limit. The head has only $911.76 of countable resources and passes the resource test; the subsequent income calculation yields $7,231."
-us,scenario_057,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required SSI receipt or eligibility to be explicitly listed instead of deriving eligibility from the head's blindness and disability under assumed take-up. The eligible head passes the resource test and receives $7,231 after income counting."
-us,scenario_057,ssi,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model treated both spouses as eligible and applied the couple rate and couple-wide exclusions. The spouse fails the SSI categorical test, so only the head's individual maximum is reduced by deemed countable income."
-us,scenario_057,ssi,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used an eligible-couple calculation because it treated both disability flags as sufficient for SSI. Only the head is categorically eligible, and the individual calculation with spousal deeming yields $7,231."
-us,scenario_057,ssi,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly stated that both spouses meet SSI disability criteria and reduced a couple maximum. PolicyEngine recognizes only the head as SSI aged, blind, or disabled, producing an individual benefit of $7,231."
-us,scenario_057,ssi,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model treated both adults as categorically eligible and calculated a federal couple benefit. The spouse receives zero, while the head's individual benefit after spousal deeming is $7,231."
-us,scenario_057,ssi,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model asserted that self-employment income exceeds the SSI maximum without applying the earned-income exclusions and correct monthly countable-income calculation. Countable income is $391.42 per month, leaving the head a positive $602.58 monthly payment."
-us,scenario_057,ssi,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model treated both spouses as eligible and subtracted income from a projected couple federal benefit rate. Only the head meets the SSI categorical criterion, requiring an individual benefit calculation with spousal deeming."
-us,scenario_057,ssi,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model incorrectly used a married-couple maximum on the premise that both spouses were SSI-disabled. The spouse fails the operative categorical test, and only the head's individual benefit is payable."
-us,scenario_057,ssi,inkling,llm_error,household_unit_or_filing_status,False,"The model started from the SSI couple rate and therefore overstated the maximum payable. The spouse is not categorically eligible; applying spousal deeming to the eligible head's individual calculation gives $7,231."
-us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, so it never performed the required calculation."
-us,scenario_057,ssi,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model assumed both spouses were SSI-eligible and used the couple federal benefit rate. Only the head passes the categorical test, and the individual benefit after spousal deeming is $7,231."
-us,scenario_057,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred SSI eligibility and payment from the small listed disability-benefit amounts instead of applying the head's blindness/disability status and assumed program take-up. The head qualifies and receives $7,231 after the resource and income tests."
-us,scenario_057,ssi,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model alternated among couple totals and separate benefits for two allegedly eligible spouses, including a payment for the spouse. SSI is individual, the spouse fails the categorical criterion, and only the head's benefit with spousal deeming is payable."
-us,scenario_057,ssi,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that income and resources eliminate SSI without applying the traced eligibility and income-counting steps. The head passes the categorical and $911.76 resource tests, and $391.42 of monthly countable income leaves $7,231 annually."
+us,scenario_057,ssi,claude-fable-5,llm_error,missing_output,False,"The model submitted a placeholder zero without applying SSI eligibility or benefit rules. The head is SSI-eligible, and reducing the individual benefit by $391.42 of monthly countable income yields $7,231 annually."
+us,scenario_057,ssi,claude-haiku-4.5,llm_error,period_annualization,False,"The model treated the annual $10,400 of self-employment income as though nearly all of it were countable instead of applying the SSI exclusions monthly and halving the remaining earned income. It also pooled the spouses into a couple calculation; the head's individual calculation has $391.42 in monthly countable income and yields $7,231."
+us,scenario_057,ssi,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated both spouses as SSI-eligible and subtracted pooled income from the eligible-couple federal benefit rate. SSI is individual here: the spouse fails PolicyEngine's aged/blind/disabled criterion, while the head receives the individual rate reduced by spousal deeming and other countable income."
+us,scenario_057,ssi,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model incorrectly granted both spouses SSI eligibility and used a $1,538 monthly couple rate. The spouse is not SSI aged/blind/disabled under the modeled rules, so only the head's individual benefit is payable after applying spousal deeming."
+us,scenario_057,ssi,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model failed to apply the SSI earned-income exclusions and 50% disregard to monthly self-employment income, causing it to treat the earnings as disqualifying. Those rules leave the head with $391.42 of monthly countable income and a positive $602.58 monthly payment."
+us,scenario_057,ssi,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated the household as an eligible SSI couple and used a projected couple federal benefit rate. The spouse does not satisfy PolicyEngine's SSI aged/blind/disabled criterion, so the correct computation uses the head's individual rate with spousal deeming, not a couple award."
+us,scenario_057,ssi,claude-sonnet-5,llm_error,other,False,"The model's reasoning actually derived a positive benefit but then submitted zero, contradicting its own calculation. It also used an eligible-couple calculation even though only the head meets the modeled SSI aged/blind/disabled criterion."
+us,scenario_057,ssi,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model correctly approached the monthly income exclusions but applied the resulting $392-level countable income against a projected couple benefit rate. Only the head is SSI-eligible, so the corresponding individual-rate calculation yields $602.58 per month rather than $1,090.58."
+us,scenario_057,ssi,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model assumed both spouses formed an eligible SSI couple and used the couple maximum. The spouse fails the modeled aged/blind/disabled criterion, leaving an individual award for the head with spousal deeming."
+us,scenario_057,ssi,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the head's earned income as categorically disqualifying without applying SSI's general exclusion, earned-income exclusion, and 50% earned-income disregard. After those rules, countable monthly income is $391.42 and SSI remains payable."
+us,scenario_057,ssi,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model explicitly used the SSI couples maximum, incorrectly treating the spouse as an eligible member of an SSI couple. PolicyEngine finds only the head SSI aged/blind/disabled and calculates an individual benefit with spousal deeming."
+us,scenario_057,ssi,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly declared both spouses SSI-eligible and subtracted income from an estimated couple federal benefit rate. The spouse is not SSI aged/blind/disabled under the modeled rules, so only the head's individual award is payable."
+us,scenario_057,ssi,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model treated both disabled inputs as establishing SSI eligibility and calculated a couple benefit. PolicyEngine's categorical test excludes the spouse, and the head must instead receive an individual benefit reduced by countable income and spousal deeming."
+us,scenario_057,ssi,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model incorrectly classified the spouses as an eligible SSI couple. Only the head passes the modeled SSI aged/blind/disabled test, so the couple maximum cannot be used."
+us,scenario_057,ssi,glm-5.2,llm_error,categorical_eligibility,False,"The model ignored the head's explicit blindness and disability and incorrectly treated $10,400 of earnings as exceeding SSI eligibility limits. It also treated absence of listed SSI receipt as dispositive despite the instruction to assume take-up; the head passes the categorical and resource tests and receives a reduced benefit."
+us,scenario_057,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model falsely stated that $912 of bank assets exceeds the SSI asset limit. PolicyEngine counts $911.76 in resources, which passes the resource test, and the income exclusions leave a positive $7,231 benefit."
+us,scenario_057,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed SSI payment as proof of no benefit, contrary to the instruction to calculate eligibility and assume take-up. The head's blindness and disability establish the categorical pathway, and the resource and income tests produce $7,231."
+us,scenario_057,ssi,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model treated both spouses as SSI-eligible and used the eligible-couple rate of $1,491 per month. The spouse fails PolicyEngine's SSI aged/blind/disabled test, so the head receives an individual benefit with spousal deeming rather than a couple award."
+us,scenario_057,ssi,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model incorrectly said both adults meet SSI disability criteria and applied a couple federal benefit rate. Only the head passes the modeled categorical test, requiring an individual benefit calculation with spousal deeming."
+us,scenario_057,ssi,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model incorrectly classified both spouses as disability-eligible for SSI and reduced a couple maximum. The spouse receives zero because the modeled aged/blind/disabled test fails, while the head receives an individual award."
+us,scenario_057,ssi,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used SSI couple eligibility and couple-rate exclusions for two eligible adults. PolicyEngine recognizes only the head as SSI aged/blind/disabled, so the benefit is calculated individually with spousal deeming."
+us,scenario_057,ssi,grok-4.3,llm_error,taxable_income_or_deductions,False,The model treated gross self-employment income as exceeding the SSI payment threshold without applying the monthly general and earned-income exclusions and the 50% earned-income disregard. Those rules reduce countable income to $391.42 per month and leave a positive head benefit.
+us,scenario_057,ssi,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied its approximately $393 monthly countable-income result to a projected couple federal benefit rate. The spouse is not SSI aged/blind/disabled under PolicyEngine, so that income must reduce the head's individual rate instead."
+us,scenario_057,ssi,grok-4.6,llm_error,household_unit_or_filing_status,False,"The model used the annual eligible-couple federal benefit rate because it assumed both spouses qualified. Only the head meets the categorical SSI test, and the correct individual calculation with spousal deeming produces $7,231."
+us,scenario_057,ssi,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model classified the household as a married SSI-eligible couple and used an estimated couple maximum. The spouse fails the modeled SSI aged/blind/disabled criterion, so only the head's individual award is calculated."
+us,scenario_057,ssi,inkling,llm_error,household_unit_or_filing_status,False,"The model subtracted approximate countable self-employment income from a $1,490 monthly couple rate. Only the head is SSI-eligible, so the applicable maximum is the individual rate and the resulting annual payment is $7,231."
+us,scenario_057,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no SSI value or explanation, so the required output could not be parsed."
+us,scenario_057,ssi,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model incorrectly treated both spouses as SSI-eligible and used the $1,491 monthly couple rate. PolicyEngine's categorical test excludes the spouse, leaving an individual benefit for the head after spousal deeming."
+us,scenario_057,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred SSI ineligibility from the small reported disability-benefit amounts instead of applying the explicit blindness and disability facts to the SSI categorical test. It also confused listed benefit receipt with computed entitlement; the head qualifies and receives $7,231."
+us,scenario_057,ssi,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model correctly approximated monthly countable earnings but deducted them from a couple federal benefit rate. The spouse does not pass PolicyEngine's SSI aged/blind/disabled test, so the head's individual rate is the proper starting amount."
+us,scenario_057,ssi,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model alternated among obsolete individual and couple rates, treated the annual $14 and $24 disability benefits as monthly in parts of its reasoning, and finally chose an unsupported amount. SSI is calculated only for the eligible head, with spousal deeming and $391.42 of monthly countable income, yielding $7,231 annually."
+us,scenario_057,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that the inputs produce no SSI without applying the head's explicit blindness and disability, the passing $911.76 resource amount, or the earned-income exclusions. Those rules establish eligibility and leave a positive $7,231 annual payment."
 us,scenario_057,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied obsolete Louisiana rules: two $4,500 personal exemptions and a 1.75% first tax bracket. Under the 2026 married-filing-jointly deduction and rate structure, the household’s approximately $9,665 of adjusted income produces zero Louisiana taxable income and therefore $0 tax before refundable credits."
 us,scenario_057,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value for state_income_tax_before_refundable_credits, violating the required output contract. The required computation yields $0 because the household has no positive Louisiana taxable income under the 2026 married-filing-jointly rules."
-us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model identified Louisiana’s 5% EITC and the childless federal EITC pathway but then discarded both instead of calculating them. Self-employment earnings qualify as earned income, producing a $664 federal EITC and a $33.20 Louisiana credit."
-us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model omitted Louisiana’s refundable EITC and incorrectly treated the absence of dependents as eliminating all refundable credits. Childless joint filers can receive the federal EITC, and Louisiana refunds 5% of the resulting $664 credit."
-us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child for Louisiana’s EITC and discounted self-employment as qualifying earned income. The childless federal EITC is $664, yielding a $33.20 Louisiana refundable EITC."
-us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model recognized the 5% Louisiana EITC but incorrectly rounded a nonzero childless federal EITC to zero. The federal credit is $664, and 5% is $33.20 rather than an amount that rounds to zero."
-us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly characterized Louisiana’s 5% EITC as nonrefundable or otherwise inapplicable. It is refundable, and the household’s $664 federal EITC generates a $33.20 state credit."
-us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no state EITC. Louisiana provides a refundable credit equal to 5% of the federal EITC, producing $33.20 from the household’s $664 federal credit."
-us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana lacks an applicable refundable EITC and treated dependents as necessary. Louisiana refunds 5% of the childless household’s $664 federal EITC, or $33.20."
-us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model applied the correct 5% Louisiana rate to an understated federal EITC of $649. The federal EITC is $664, so the state credit is $33.20."
-us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,The model applied the correct 5% Louisiana rate but understated the federal EITC as $630. Using the $664 federal EITC yields $33.20.
-us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits Louisiana’s refundable EITC pathway. The household receives a $664 federal EITC from its qualifying self-employment earnings, and Louisiana refunds 5%, or $33.20."
-us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly found no applicable Louisiana refundable credit. The childless household qualifies for a $664 federal EITC, which generates a $33.20 refundable Louisiana EITC."
+us,scenario_057,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model recognized Louisiana's 5% EITC but incorrectly discarded the childless federal EITC and invoked an unstated microsimulation result of zero. The household's self-employment earnings generate a $664 federal EITC, yielding a $33.20 Louisiana credit."
+us,scenario_057,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC and incorrectly treated dependents or special expenses as necessary. A childless married household can receive the federal EITC, and Louisiana refunds 5% of its $664 federal credit."
+us,scenario_057,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly required a qualifying child or related expense for Louisiana's EITC and failed to treat self-employment income as qualifying earned income. The childless federal EITC is $664, producing a $33.20 Louisiana EITC."
+us,scenario_057,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model acknowledged the 5% Louisiana EITC but incorrectly rounded a nonzero childless credit to zero. The federal EITC is $664, and 5% of that amount is $33.20."
+us,scenario_057,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly characterized Louisiana's 5% EITC as nonrefundable or otherwise inapplicable. It is refundable, and the household's $664 federal EITC generates a $33.20 state credit."
+us,scenario_057,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no refundable state EITC in 2026. Louisiana provides a refundable credit equal to 5% of the $664 federal EITC, or $33.20."
+us,scenario_057,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana lacks an applicable refundable EITC and treated the absence of dependents as disqualifying. Louisiana refunds 5% of the childless household's $664 federal EITC, totaling $33.20."
+us,scenario_057,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model applied the correct 5% Louisiana rate to an understated federal EITC of $649. The federal EITC is $664, so the state credit is $33.20 rather than $32.45."
+us,scenario_057,state_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,The model applied the correct 5% Louisiana rate but understated the federal EITC as $630. Applying 5% to the correct $664 federal EITC yields $33.20.
+us,scenario_057,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer omits the refundable Louisiana EITC triggered by the household's federal EITC. The $664 federal credit produces a $33.20 Louisiana credit at the 5% rate.
+us,scenario_057,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly declared the household ineligible for every Louisiana refundable credit. Its childless federal EITC is $664, which qualifies it for a $33.20 refundable Louisiana EITC."
 us,scenario_057,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,The model used the correct 5% state rate but understated the federal EITC as $649. The correct $664 federal EITC produces a $33.20 Louisiana credit.
-us,scenario_057,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model applied the childless EITC age rule to the 20-year-old spouse as though both spouses had to meet the minimum age. The 43-year-old head satisfies the joint-filer age condition, so the $664 federal EITC produces a $33.20 Louisiana credit."
-us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The zero answer omits the refundable Louisiana EITC triggered by the household’s federal EITC. A $664 federal credit yields $33.20 at Louisiana’s 5% rate.
-us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to apply Louisiana’s refundable EITC to the listed self-employment earnings. The resulting $664 federal EITC generates a $33.20 state credit.
-us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly treated a qualifying child as necessary for Louisiana’s refundable EITC. The household qualifies for the childless federal EITC of $664, making the Louisiana credit $33.20."
-us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model overlooked that self-employment earnings trigger the childless federal EITC and therefore Louisiana’s refundable match. Five percent of the $664 federal credit is $33.20.
-us,scenario_057,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted the Louisiana refundable EITC. The household’s $664 federal EITC produces a $33.20 Louisiana credit at the 5% match rate.
-us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly concluded that Louisiana has no refundable credit applicable to this household. Louisiana’s refundable EITC equals 5% of the $664 federal EITC, or $33.20."
-us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model failed to include Louisiana’s refundable EITC among the applicable state credits. The household’s $664 federal EITC generates a $33.20 state credit.
-us,scenario_057,state_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model incorrectly computed the federal EITC as zero despite qualifying self-employment earnings and an age-eligible head. The federal EITC is $664, so Louisiana’s 5% credit is $33.20."
+us,scenario_057,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly treated the spouse's age of 20 as disqualifying the married couple from the childless EITC. The head is 43, the household receives a $664 federal EITC, and Louisiana refunds 5% of it, or $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to trigger Louisiana's refundable EITC from the household's self-employment earnings. The $664 federal EITC generates a $33.20 state credit.
+us,scenario_057,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model omitted Louisiana's refundable EITC despite the listed self-employment income. The resulting $664 federal EITC carries through at 5%, producing $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required children for Louisiana's refundable EITC. This childless household qualifies for a $664 federal EITC, of which Louisiana refunds 5%, or $33.20."
+us,scenario_057,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model failed to recognize that the household's federal EITC automatically supplies the base for Louisiana's refundable EITC. Five percent of the $664 federal credit is $33.20.
+us,scenario_057,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable state credit applies and omitted Louisiana's 5% EITC. The household's $664 federal EITC produces a $33.20 state refund.
+us,scenario_057,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana provides no applicable refundable income-tax credit. Louisiana's refundable EITC equals 5% of this household's $664 federal EITC, or $33.20."
+us,scenario_057,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model omitted Louisiana's refundable EITC from the applicable state credits. The household receives a $664 federal EITC, making the Louisiana credit $33.20 at the statutory 5% rate."
+us,scenario_057,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model incorrectly computed the federal EITC as zero despite qualifying self-employment earnings and childless eligibility. The federal EITC is $664, so Louisiana's 5% credit is $33.20."
 us,scenario_057,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, violating the required output contract."
-us,scenario_057,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model doubled Louisiana’s EITC percentage by applying a 10% rate. The applicable rate is 5% of the $664 federal EITC, yielding $33.20."
-us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model falsely stated that Louisiana has no individual income tax and consequently omitted its refundable EITC. Louisiana’s 5% match of the $664 federal EITC equals $33.20.
-us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model falsely stated that Louisiana has no state individual income tax and omitted the refundable Louisiana EITC. The credit is 5% of $664, or $33.20."
-us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model failed to recognize that this childless household’s self-employment earnings qualify it for a federal EITC and Louisiana’s refundable match. The $664 federal credit produces $33.20 at the 5% state rate.
+us,scenario_057,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model doubled Louisiana's EITC percentage by applying a 10% rate. The applicable rate is 5% of the $664 federal EITC, producing $33.20."
+us,scenario_057,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no individual income tax. Louisiana has an individual income tax and a refundable EITC equal to 5% of the household's $664 federal EITC, yielding $33.20."
+us,scenario_057,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model incorrectly tied refundable-credit eligibility to positive state tax liability, qualifying expenses, or children. Louisiana's refundable EITC is available against the household's $664 federal EITC and equals $33.20 even with zero pre-credit liability."
+us,scenario_057,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly stated that Louisiana has no individual income tax. Louisiana's refundable EITC equals 5% of the $664 federal EITC, resulting in $33.20."
+us,scenario_057,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted the refundable Louisiana EITC generated by the household's federal EITC. The correct state credit is 5% of $664, or $33.20."
 us,scenario_059,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the separate 2026 enhanced senior deduction for each spouse, so it incorrectly left positive taxable income after its standard and age-based deductions. It also overstated taxable Social Security instead of applying the worksheet result of $6,129.20."
 us,scenario_059,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an $18,100 joint standard deduction, omitting the full 2026 married-filing-jointly standard deduction, both age-65 additions, and both enhanced senior deductions. With the correctly calculated $31,729.20 AGI, the applicable deductions eliminate taxable income."
 us,scenario_059,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model substituted an obsolete personal-exemption framework and undersized standard and senior deductions for the applicable 2026 deduction rules. The joint standard deduction, two age-based additions, and two enhanced senior deductions exceed $31,729.20 of AGI, leaving no taxable income."
@@ -3753,22 +3942,23 @@ us,scenario_060,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The
 us,scenario_060,snap,claude-opus-4.7,llm_error,other,False,"The model correctly counted the $45,600 of veterans benefits and calculated net income of about $3,994 per month, which exceeds the one-person net-income limit. It then discarded its own eligibility conclusion and submitted $2,496 without any computation supporting that amount; an income-ineligible household receives $0, not a minimum or partial allotment."
 us,scenario_060,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $45,600 of veterans benefits from SNAP countable unearned income. Including those benefits prevents the rent deduction from reducing net income to zero and leaves the household above the one-person net-income eligibility limit."
 us,scenario_060,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model’s $1,431.25 monthly gross-income figure omits the $3,800 monthly veterans benefit from SNAP countable income. That omission lets the medical and shelter deductions erase all net income; with veterans benefits included, countable net income remains above the eligibility threshold and SNAP equals $0."
-us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the separate $28,800 survivor-benefits input as ordinary taxable income, which drove provisional income, taxable Social Security, and AGI sharply upward. It also assigned qualifying-surviving-spouse filing status from the surviving-spouse flag without the required dependent-child facts; excluding the nontaxable survivor-benefits input and applying the proper deductions leaves zero tax."
-us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included the $28,800 survivor-benefits input as taxable income, producing the erroneous $63,245 AGI and causing it to tax 85% of Social Security. It then used only a single-filer standard deduction and omitted the deductions that eliminate the remaining taxable income."
-us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model wrongly assumed a 2026 TCJA sunset, restored a personal exemption, and applied obsolete 10%/15%/25% brackets instead of 2026 current law. It also treated the $28,800 survivor-benefits input as taxable income, so both its tax base and its rate schedule were wrong."
-us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $21,312 Social Security benefit and the separate $28,800 survivor-benefits input as AGI while subtracting the positive $4,793 farm-rent income, an internally incorrect income construction. Its stated $29,449 taxable income also cannot produce $178 of tax under any cited bracket calculation; the correct taxable-income computation yields zero liability."
-us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the separate survivor-benefits amount from AGI but applied an obsolete post-sunset combination of standard deduction, age addition, and personal exemption totaling only about $15,800. The applicable 2026 deductions absorb the approximately $17,323 AGI, leaving no taxable income rather than $1,523."
-us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly included the $28,800 survivor-benefits input as taxable income and consequently treated 85% of Social Security as taxable, inflating AGI to about $63,245. It also understated the applicable deductions by using an approximate $15,400 total, whereas the correct income-and-deduction computation leaves no taxable income."
-us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model treated the $28,800 survivor-benefits input as taxable, then applied a fictional post-TCJA-sunset deduction structure and obsolete 10%/15%/25% brackets. Excluding that benefit from taxable income and using the applicable 2026 deductions eliminates taxable income entirely."
-us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $6,546.75 answer implies the same inflated tax base obtained by adding the $28,800 survivor-benefits input and a large taxable Social Security amount to pension, rent, and interest. The separate survivor-benefits amount is not taxable AGI, and the applicable deductions reduce the actual tax base to zero."
-us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly included taxable survivor income in its federal tax base, incorrectly treating the separate $28,800 survivor-benefits input as taxable AGI. That inclusion also inflated taxable Social Security; after excluding it and applying the age-related and senior deductions, no taxable income remains."
-us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly counted the $28,800 survivor-benefits input as taxable income and therefore overstated both AGI and the taxable share of Social Security. The single, age-based, senior, and other applicable deductions reduce the correctly constructed income base to zero taxable income."
-us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model added the $28,800 survivor-benefits input to AGI and assumed a TCJA sunset with restored personal exemptions and pre-TCJA rates. Both choices are wrong for the 2026 calculation; the applicable current-law deductions against the correctly defined income yield zero taxable income."
-us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model's $63,245 AGI comes from incorrectly taxing the separate $28,800 survivor-benefits input and, through that inflated provisional income, 85% of Social Security. It then used a post-TCJA personal exemption and obsolete 10%/15% brackets instead of the applicable 2026 deduction and rate rules that produce zero liability."
-us,scenario_062,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $28,800 survivor-benefits input in AGI, which also caused it to tax about $18,115 of Social Security. Its deduction calculation therefore began from an inflated $63,245 AGI; the correctly constructed income is fully offset by applicable deductions."
-us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, so it failed the required output contract before any substantive tax calculation could be evaluated."
-us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the separate $28,800 survivor-benefits input and calculated approximately $17,323 of AGI, but it understated the applicable 2026 deductions by relying on a restored-exemption framework. Those deductions eliminate all taxable income, so the remaining $1,820 tax base and $182 tax do not arise."
-us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly decided that the $28,800 survivor-benefits input was taxable and added it to AGI, which raised provisional income enough for it to tax 85% of Social Security. That benefit is not taxable AGI; excluding it and applying the applicable deductions leaves zero taxable income and makes the elderly-credit phaseout discussion immaterial."
+us,scenario_062,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated the surviving-spouse indicator as qualifying-widow filing status, even though that status is not established by the listed facts, and used married-joint deductions and brackets. It also improperly included the separate $28,800 survivor benefit in taxable AGI, which inflated both AGI and taxable Social Security."
+us,scenario_062,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added the nontaxable $28,800 survivor benefit to AGI, causing it to tax 85% of Social Security and report AGI of $63,245. It also omitted the 2026 senior deduction, whereas the correct taxable items and applicable deductions leave no taxable income."
+us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly taxed the $28,800 survivor benefit and consequently treated 85% of Social Security as taxable. It also applied an obsolete TCJA-sunset regime instead of the applicable 2026 standard, age-based, and senior deductions that eliminate taxable income."
+us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated Social Security and the separate survivor benefit as taxable income while excluding taxable farm rent, then produced a tax amount that does not follow from its own stated $29,449 of taxable income. It also omitted the 2026 senior deduction that reduces the correctly computed taxable income to zero."
+us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the separate survivor benefit and arrived near the correct $17,323 AGI, but it applied a supposed TCJA-expiration deduction regime and omitted the 2026 senior deduction. The full applicable deductions exceed that AGI, leaving zero taxable income rather than about $1,523."
+us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model added the nontaxable $28,800 survivor benefit to AGI and used that inflated provisional income to tax 85% of Social Security. It also understated the applicable deductions by omitting the 2026 senior deduction, producing taxable income where none remains."
+us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly included the $28,800 survivor benefit in AGI, which drove Social Security taxation to 85%. It then used an obsolete personal-exemption framework and omitted the 2026 senior deduction that, together with the standard and age-based deductions, eliminates taxable income."
+us,scenario_062,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its positive tax calculation implies that it treated the separate survivor benefit and a large Social Security share as taxable, as stated in its income list. The correct federal income base is about $17,323 before deductions, and the applicable 2026 deductions reduce it to zero."
+us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly included the separate survivor benefit and a taxable Social Security share in federal taxable income. Excluding that survivor benefit produces only about $17,323 of AGI, which is fully absorbed by the standard, age-based, and senior deductions."
+us,scenario_062,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the separate $28,800 survivor benefit as taxable and thereby overstated both AGI and taxable Social Security. With that benefit excluded, the applicable single, age-based, and senior deductions reduce taxable income to zero."
+us,scenario_062,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model included the nontaxable $28,800 survivor benefit in AGI and consequently taxed 85% of Social Security. It also substituted an obsolete post-TCJA-sunset personal-exemption regime for the applicable 2026 deductions, including the senior deduction."
+us,scenario_062,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly counted the $28,800 survivor benefit as taxable, raising provisional income enough to classify $18,115 of Social Security as taxable. It also used a restored-personal-exemption framework and omitted the 2026 senior deduction, while the correct deductions eliminate all taxable income."
+us,scenario_062,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added the separate $28,800 survivor benefit to AGI and therefore taxed approximately $18,115 of Social Security. It also omitted the 2026 senior deduction; after excluding the survivor benefit and applying all deductions, taxable income is zero."
+us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, so it failed the required structured-output contract."
+us,scenario_062,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the separate survivor benefit and computed approximately $17,323 of AGI, but it omitted the 2026 senior deduction. That deduction, combined with the standard and age-based deductions, eliminates the $1,820 of taxable income it reported."
+us,scenario_062,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model improperly included the $28,800 survivor benefit in AGI and used the resulting inflated provisional income to tax 85% of Social Security. Although it applied the senior deduction, removing the nontaxable survivor benefit leaves deductions greater than taxable AGI and therefore zero tax."
+us,scenario_062,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model expressly assumed that the $28,800 survivor-benefits input was taxable military Survivor Benefit Plan income without any fact establishing that characterization. This inflated provisional income, caused 85% of Social Security to be taxed, and was compounded by omission of the 2026 senior deduction; the correct taxable income after deductions is zero."
 us,scenario_062,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_062,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model acknowledged that the bank assets exceed the SSI-related resource limit, then invented medically needy and spousal-protection eligibility without facts establishing either pathway. Surviving-spouse status does not supply a community spouse or override the absence of a qualifying Medicaid category."
 us,scenario_062,head_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the Social Security survivor benefit from SSI-related countable income and then characterized $16,330 of its remaining annual income as below the aged/disabled standard. The head receives no SSI and qualifies under no aged/disabled Medicaid category."
@@ -3777,12 +3967,13 @@ us,scenario_062,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligib
 us,scenario_062,head_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model conflated Medicare Savings Program or SSI-linked coverage with full Medicaid eligibility and assumed deductions from medical expenses would create eligibility. The head receives zero SSI and qualifies under no Medicaid category, so age, disability, Medicare eligibility, and health expenses do not produce Medicaid eligibility."
 us,scenario_062,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_062,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $4,793 of farm rent income as self-employment income. Farm rent does not create net earnings from self-employment here, so the correct tax base is zero rather than $4,793 and self-employment tax is $0."
-us,scenario_064,child1_chip_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model tested Child 1's personal income of zero instead of the income of the child's applicable household. Applying the household's approximately $98,810 income causes the child to fail Wisconsin CHIP's income test."
-us,scenario_064,child1_chip_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly treated Wisconsin CHIP as available up to 300% of the applicable federal poverty level and placed the family below that ceiling. The applicable CHIP income test excludes this household at approximately $98,810."
-us,scenario_064,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 306% FPL ceiling as the controlling Wisconsin CHIP threshold and incorrectly classified the household as below it. Under the modeled Wisconsin CHIP income test, approximately $98,810 exceeds the applicable threshold."
-us,scenario_064,child1_chip_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model invented an upper CHIP income band that it placed above the household's income. Child 1 is under 19 and not Medicaid-eligible, but the separate Wisconsin CHIP income test also fails at approximately $98,810."
-us,scenario_064,child1_chip_eligible,grok-build-0.1,llm_error,thresholds_rates,False,"The model misapplied a 300% FPL ceiling and then added an employer-coverage affordability exception that does not cure the failed modeled income test. At approximately $98,810, the household exceeds the applicable Wisconsin CHIP threshold regardless of its premium-to-income calculation."
-us,scenario_064,child1_chip_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly used a 300% FPL threshold for a five-person household and concluded that deductions and losses brought MAGI below it. The applicable Wisconsin CHIP income test places the household's approximately $98,810 income above the eligibility threshold."
+us,scenario_064,child1_chip_eligible,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model tested the child's own reported income instead of the income of the child's CHIP eligibility household. PolicyEngine attributes approximately $98,810 of household income to the eligibility test, which exceeds the applicable Wisconsin CHIP ceiling."
+us,scenario_064,child1_chip_eligible,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly classified the household as below a 300% FPL BadgerCare Plus limit. Under the applicable PolicyEngine Wisconsin CHIP calculation, approximately $98,810 exceeds the child's CHIP income ceiling."
+us,scenario_064,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 306% FPL threshold and incorrectly placed the household below it. PolicyEngine's applicable Wisconsin CHIP income test places approximately $98,810 above the ceiling, so the age-qualified child fails on income."
+us,scenario_064,child1_chip_eligible,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model incorrectly placed the child within Wisconsin's upper CHIP income band. PolicyEngine places the household's approximately $98,810 of income above the applicable CHIP band, leaving the child ineligible for both Medicaid and CHIP."
+us,scenario_064,child1_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model's calculation of household MAGI as about 297% FPL and below a 306% ceiling misapplied the applicable CHIP threshold calculation. PolicyEngine's Wisconsin test places approximately $98,810 above the CHIP income ceiling."
+us,scenario_064,child1_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used approximately $106,225, treated losses and contributions as reducing CHIP MAGI, and then classified that amount as 273% FPL below a 300% limit. PolicyEngine calculates approximately $98,810 for the relevant household-income measure but still places it above the applicable Wisconsin CHIP ceiling; the asserted affordability exception for employer coverage does not overcome the failed income test."
+us,scenario_064,child1_chip_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that pre-tax contributions and claimed allowable losses reduce household MAGI below a 300% FPL CHIP threshold. PolicyEngine's applicable income computation produces approximately $98,810 and places it above Wisconsin's CHIP ceiling, so the child fails the income criterion despite being under 19."
 us,scenario_064,child1_medicaid_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,The model asserted eligibility under a 300% FPL child limit without calculating that the household is at 322% FPL. It also invoked unspecified alternative pathways even though Child 1 qualifies through none of them and has Medicaid category NONE.
 us,scenario_064,child1_medicaid_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated the MAGI reduction from the farm loss, capital loss, and unspecified other deductions, concluding that income fell below its 301% FPL threshold. The resulting MAGI level is 322% FPL, so the child fails the applicable income test and has no alternative Medicaid category."
 us,scenario_064,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model replaced the child’s MAGI household-income test with a nonexistent separate child-only test based on the child’s $0 personal income. The applicable household MAGI is 322% FPL, above the Wisconsin child Medicaid limit, and no other categorical pathway applies."
@@ -3791,72 +3982,77 @@ us,scenario_064,child1_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,Fal
 us,scenario_064,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child1_medicaid_eligible, so it failed the required output contract."
 us,scenario_064,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated age 12 as within WIC's eligible child age range. WIC covers children only through age four, so Child 1 fails categorical eligibility regardless of income; the household also exceeds the income limit."
 us,scenario_064,dependent1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly attributed the head and spouse's tax-unit MAGI to dependent1. PolicyEngine calculates dependent1's relevant MAGI income level as 0.00 times FPL, which satisfies Wisconsin's ACA adult expansion income test."
-us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied an asset test using assets listed for the head and household, even though dependent1 qualifies through the MAGI-based ACA adult category, which has no asset test. At 0.00 times FPL, dependent1 passes the expansion-adult income test."
-us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model substituted the household's aggregate income for dependent1's person-level MAGI income measure. Dependent1's modeled MAGI is 0.00 times FPL, below Wisconsin's expansion-adult threshold."
-us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,The model assigned the head's substantial income to dependent1 under both MAGI and disability pathways. PolicyEngine places dependent1 in the ACA adult category and calculates the relevant MAGI income level at 0.00 times FPL.
-us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,The model treated the full household's income as dependent1's Medicaid income. The applicable adult-expansion calculation gives dependent1 a MAGI income level of 0.00 times FPL.
-us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model aggregated the head's wages, distributions, and business income into dependent1's Medicaid MAGI. PolicyEngine instead computes dependent1's relevant MAGI income level as 0.00 times FPL and finds eligibility through Wisconsin's ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,The model compared aggregate household income with Wisconsin's adult limit instead of using dependent1's modeled MAGI income level. That person-level measure is 0.00 times FPL and passes the ACA adult test.
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The answer incorrectly implies that income counted for dependent1 exceeds Wisconsin's Medicaid limit. PolicyEngine calculates dependent1's MAGI income level at 0.00 times FPL, qualifying the person through the adult expansion category."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used the broader household's income as dependent1's Medicaid income. Dependent1's applicable MAGI income level is 0.00 times FPL, so the ACA adult income test is satisfied."
-us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,The model denied eligibility based on household income without applying PolicyEngine's person-level MAGI result. Dependent1 is at 0.00 times FPL and qualifies under Wisconsin's ACA adult category.
-us,scenario_064,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the head's wages, partnership income, and retirement distributions in dependent1's MAGI and also treated employer-sponsored insurance as disqualifying. PolicyEngine calculates dependent1 at 0.00 times FPL, and existing employer coverage does not negate Medicaid eligibility under the adult expansion test."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,The model used the household's high income as a shortcut for dependent1's eligibility. The correct person-level adult-expansion calculation yields MAGI of 0.00 times FPL and eligibility.
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated additional disability-related facts as necessary and then denied eligibility under a simplified household-income test. Disability details are not required for the applicable pathway because dependent1 qualifies as an under-65 ACA adult at 0.00 times FPL.
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model attributed the family's annual income to dependent1 solely because the person is a tax dependent. PolicyEngine's applicable MAGI calculation places dependent1 at 0.00 times FPL, below Wisconsin's adult expansion limit."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,asset_resource,False,"The model invoked income and resource rules without applying the MAGI adult pathway. That pathway has no resource test, and dependent1's MAGI income level is 0.00 times FPL."
-us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model treated the available income and assets as barriers to dependent1's Medicaid eligibility. PolicyEngine qualifies dependent1 through the MAGI-based adult category, which has no asset test, at an income level of 0.00 times FPL."
-us,scenario_064,dependent1_medicaid_eligible,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model's denial implies that it assigned other household members' income to dependent1. The applicable MAGI result is 0.00 times FPL, below Wisconsin's adult expansion threshold."
-us,scenario_064,dependent1_medicaid_eligible,grok-4.5,llm_error,categorical_eligibility,False,"The model routed dependent1 through the non-MAGI aged, blind, or disabled resource test and also asserted failure of the MAGI adult limit. PolicyEngine instead establishes eligibility through the ACA adult category at 0.00 times FPL, where household assets are irrelevant."
-us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model calculated roughly 273% FPL by aggregating the head's income and then applied an ABD asset limit. PolicyEngine's relevant calculation for dependent1 is 0.00 times FPL under the ACA adult category, which does not apply an asset test."
-us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for the requested variable. It therefore failed the required output contract before any Medicaid rule was evaluated.
-us,scenario_064,dependent1_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The unexplained denial omits the controlling adult-expansion derivation. Dependent1 is age 27 with MAGI at 0.00 times FPL, which satisfies Wisconsin's ACA adult category."
-us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,The model included the head's wages and other income in dependent1's Medicaid MAGI and stated a 100% FPL adult limit. PolicyEngine calculates dependent1 at 0.00 times FPL and applies Wisconsin's ACA adult expansion category.
-us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,llm_error,household_unit_or_filing_status,False,The model asserted that dependent1's income is too high despite no income being listed for that person. PolicyEngine's adult-expansion calculation yields a MAGI income level of 0.00 times FPL and eligibility.
+us,scenario_064,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly attributed the head and spouse's tax-unit MAGI to dependent1. PolicyEngine calculates dependent1's applicable MAGI income level as 0.00 times FPL, which passes Wisconsin's adult-expansion threshold."
+us,scenario_064,dependent1_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model applied an asset test using resources owned by the household to deny eligibility. Dependent1 qualifies through the MAGI-based ACA adult category, which uses the 0.00-times-FPL MAGI result and does not impose that disability-related resource test."
+us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,The model substituted the household's aggregate income for dependent1's category-specific MAGI income level. The applicable adult-expansion calculation produces 0.00 times FPL and therefore establishes eligibility.
+us,scenario_064,dependent1_medicaid_eligible,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model denied both MAGI- and disability-based eligibility from the household's total income without applying dependent1's adult-expansion calculation. That calculation assigns dependent1 a MAGI income level of 0.00 times FPL, below the expansion limit."
+us,scenario_064,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model treated the adult dependent's eligibility income as the household's high aggregate income. PolicyEngine's applicable ACA adult calculation instead yields 0.00 times FPL for dependent1, satisfying the income test."
+us,scenario_064,dependent1_medicaid_eligible,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model summed the head's wages, partnership income, distributions, and other amounts into dependent1's Medicaid MAGI comparison. Dependent1's adult-expansion MAGI income level is 0.00 times FPL, so that household-level aggregation produced the wrong eligibility result."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a 100% FPL adult limit and asserted excess household income. The applicable ACA adult category extends to 138% FPL, and dependent1's calculated MAGI income level is 0.00 times FPL."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model concluded that income exceeded Wisconsin's limit instead of applying dependent1's calculated adult-category MAGI. That calculation yields 0.00 times FPL, safely below the expansion threshold."
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,The model used household income to deny dependent1's adult Medicaid eligibility. The relevant MAGI income level for dependent1 is 0.00 times FPL and passes Wisconsin's adult-expansion test.
+us,scenario_064,dependent1_medicaid_eligible,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,The model denied eligibility based on aggregate household income. PolicyEngine's category-specific calculation places dependent1 at 0.00 times FPL under the ACA adult pathway.
+us,scenario_064,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model counted the head's wages, partnership income, and retirement distributions as dependent1's Medicaid MAGI and also treated employer-sponsored insurance as disqualifying. Dependent1's applicable MAGI level is 0.00 times FPL, and existing employer coverage does not defeat adult-expansion eligibility."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,The answer's high-income-household shortcut skipped dependent1's individual adult-expansion MAGI result. That result is 0.00 times FPL and satisfies Wisconsin's expansion income limit.
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model made eligibility depend on additional disability details and a simplified household-income assumption. Dependent1 qualifies through the ACA adult category, not a disability pathway, because their MAGI income level is 0.00 times FPL."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,The model treated tax-dependent status as requiring the family's annual income to be used as dependent1's adult Medicaid income. The applicable PolicyEngine calculation instead yields a MAGI income level of 0.00 times FPL for dependent1.
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,asset_resource,False,"The model invoked income and resource rules without applying the qualifying MAGI adult pathway. Under that pathway dependent1 is at 0.00 times FPL, and an ABD-style resource test does not bar eligibility."
+us,scenario_064,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model used the household's income and assets to reject Medicaid eligibility. Dependent1 qualifies under the MAGI-based adult-expansion category at 0.00 times FPL, where those household assets are not a disqualifying resource test."
+us,scenario_064,dependent1_medicaid_eligible,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model asserted that income exceeded the Medicaid threshold without using dependent1's applicable MAGI result. The adult-category result is 0.00 times FPL, below Wisconsin's expansion threshold."
+us,scenario_064,dependent1_medicaid_eligible,grok-4.5,llm_error,asset_resource,False,The model focused on the non-MAGI ABD resource test and also incorrectly concluded that dependent1 failed the MAGI adult limit. The ACA adult pathway independently establishes eligibility because dependent1's MAGI income level is 0.00 times FPL.
+us,scenario_064,dependent1_medicaid_eligible,grok-4.6,llm_error,household_unit_or_filing_status,False,"The model counted dependent1 with the high-income household and supplemented that error with an irrelevant ABD resource test. PolicyEngine's MAGI adult calculation places dependent1 at 0.00 times FPL, so the expansion pathway succeeds."
+us,scenario_064,dependent1_medicaid_eligible,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model assigned dependent1 a household MAGI near 273% FPL and then applied an ABD asset limit. The applicable adult-expansion MAGI level is 0.00 times FPL, and that successful MAGI pathway is not defeated by household assets."
+us,scenario_064,dependent1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for the requested variable. A valid response had to report eligible because dependent1's ACA adult-category MAGI income level is 0.00 times FPL.
+us,scenario_064,dependent1_medicaid_eligible,minimax-m3,llm_error,other,False,"The unexplained denial skipped the controlling adult-expansion computation. Dependent1 is age 27 and has a MAGI income level of 0.00 times FPL, which satisfies the category's income test."
+us,scenario_064,dependent1_medicaid_eligible,ox-alpha,llm_error,asset_resource,False,"The model incorrectly assigned the family's high income to dependent1 and treated household bank and stock assets as disqualifying under SSI-related rules. Dependent1 qualifies through the separate MAGI adult category at 0.00 times FPL, without that resource test."
+us,scenario_064,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used an approximately 100% FPL limit and counted the head's income as dependent1's Medicaid MAGI. Wisconsin's applicable ACA adult category reaches 138% FPL, and dependent1's calculated income level is 0.00 times FPL."
+us,scenario_064,dependent1_medicaid_eligible,qwen3.8-max,llm_error,household_unit_or_filing_status,False,The model asserted that dependent1 personally had excessive income even though no income was listed for them. The adult-expansion calculation yields a MAGI income level of 0.00 times FPL and therefore establishes eligibility.
 us,scenario_064,dependent1_medicare_eligible,claude-fable-5,llm_error,age_disability,False,The model converted the listed disability flag directly into SSDI-based Medicare eligibility. It failed to require SSDI entitlement for the applicable waiting period or another qualifying condition such as ALS or end-stage renal disease.
 us,scenario_064,dependent1_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly treated disability as sufficient for Medicare eligibility regardless of age. A 27-year-old needs a distinct under-65 Medicare pathway, and no SSDI entitlement, qualifying duration, ALS, or end-stage renal disease was listed."
 us,scenario_064,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model stated the SSDI entitlement and 24-month requirements but then replaced them with the bare disability flag. Constant disability status does not establish receipt of SSDI or completion of Medicare's waiting period, and unlisted SSDI, ALS, and end-stage renal disease statuses are false."
 us,scenario_064,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated reported disability as an automatic under-65 Medicare qualification. It omitted the required SSDI-entitlement pathway and its duration requirement, as well as the alternative ALS or end-stage renal disease pathways."
-us,scenario_064,dependent2_chip_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model treated age 18 as satisfying Wisconsin CHIP’s age test and awarded eligibility from an asserted 300% FPL limit. Under the modeled rule, dependent2 fails the CHIP age criterion at 18, so the 299% FPL calculation cannot make the person eligible."
-us,scenario_064,dependent2_chip_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model applied a generic under-19 age rule and proceeded directly to a 306% FPL income threshold. PolicyEngine’s Wisconsin CHIP age test excludes this 18-year-old, who is also not Medicaid-eligible."
-us,scenario_064,dependent2_chip_eligible,gpt-5.5,llm_error,age_disability,False,"The model explicitly classified age 18 as within the child CHIP age limit. The modeled Wisconsin rule excludes dependent2 at age 18, so being within an asserted child income limit does not establish eligibility."
-us,scenario_064,dependent2_chip_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model applied an under-19 age rule and then placed dependent2 in an upper CHIP income band above Medicaid. The engine instead excludes the person under CHIP’s age criterion at age 18 and returns no Medicaid eligibility.
-us,scenario_064,dependent2_chip_eligible,grok-build-0.1,llm_error,age_disability,False,"The model treated age 18 as satisfying CHIP’s under-19 rule and relied on its 273% FPL estimate plus an ESI-affordability argument. The modeled Wisconsin CHIP age test excludes dependent2 at 18, so neither the income estimate nor the treatment of employer coverage changes the result."
-us,scenario_064,dependent2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for dependent2_chip_eligible. It therefore failed the required output contract instead of returning the engine-derived value 0.
+us,scenario_064,dependent2_chip_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model treated age 18 as satisfying Wisconsin's CHIP child-age rule and decided eligibility solely from its 299% FPL estimate. Under the modeled 2026 rule, dependent 2 fails the CHIP age criterion, regardless of that income comparison."
+us,scenario_064,dependent2_chip_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model incorrectly applied a generic “under 19” test to classify an 18-year-old as age-eligible. PolicyEngine's Wisconsin rule excludes dependent 2 at age 18, so comparing income with a 306% FPL threshold cannot establish CHIP eligibility."
+us,scenario_064,dependent2_chip_eligible,gpt-5.5,llm_error,age_disability,False,"The model misapplied the CHIP child-age limit by treating age 18 as qualifying. Dependent 2 fails the modeled Wisconsin age criterion, and Medicaid eligibility separately evaluates to NONE."
+us,scenario_064,dependent2_chip_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model wrongly placed dependent 2 in the CHIP upper-income band after treating age 18 as under the applicable child-age cutoff. The modeled Wisconsin rule rejects CHIP eligibility on age, while the Medicaid pathway returns NONE."
+us,scenario_064,dependent2_chip_eligible,grok-4.6,llm_error,age_disability,False,"The model substituted a generic under-19 age rule for PolicyEngine's applicable Wisconsin age test. Because dependent 2 is 18 and fails that test, its estimated 297% FPL and 306% ceiling comparison cannot produce CHIP eligibility."
+us,scenario_064,dependent2_chip_eligible,grok-build-0.1,llm_error,age_disability,False,"The model incorrectly treated age 18 as CHIP-qualifying, then relied on its 273% FPL estimate and an ESI-affordability analysis. Dependent 2 fails the modeled age criterion before those income and coverage considerations can establish eligibility, and Medicaid eligibility is NONE."
+us,scenario_064,dependent2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no answer for dependent2_chip_eligible. The required parsed output was 0 because dependent 2 fails the CHIP age criterion and has Medicaid category NONE.
 us,scenario_064,dependent2_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 301% FPL children's Medicaid threshold and asserted that family size brought dependent2 under it. The engine computes MAGI at 3.22 times FPL, above the applicable limits, so dependent2 receives no Medicaid category."
 us,scenario_064,dependent2_medicaid_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model used an incorrect Wisconsin Medicaid cutoff of 306% FPL, added a 5% disregard to that cutoff, and compared household income with an independently estimated family-of-five poverty amount. The engine's MAGI calculation is 3.22 times FPL and yields no qualifying Medicaid category for dependent2."
 us,scenario_064,dependent2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required output entirely. The eligibility derivation yields 0 because dependent2 is at 3.22 times FPL and qualifies through none of Wisconsin's Medicaid pathways.
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached nearly the correct taxable income but used approximate 2026 bracket thresholds and tax arithmetic, calculating about $7,630 instead of $7,639.29. It then reported $5,959 despite its own stated subtraction of $3,200 from $7,630, which equals about $4,430."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 single-filer-sized standard deduction despite identifying the return as married filing jointly, and it double-counted the overtime premium by adding it separately to wages that already include overtime. It also omitted the $13,899.29 overtime deduction and did not compute the specified $3,200 dependent credits coherently."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,899.29 overtime deduction, leaving taxable income near $80,733 instead of $67,794.08. It also invented self-employment tax and its half-tax deduction despite the farm loss offset in the traced computation, then split the CTC into refundable and nonrefundable portions even though all $3,200 was usable against regular tax."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model omitted the $13,899.29 overtime deduction and then added self-employment tax to an output expressly limited to federal income tax after nonrefundable credits. Its stated components also do not arithmetically produce $14,959."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,899.29 overtime deduction and substituted an unsupported $3,900 QBI deduction for the traced $510 amount. It also used only $2,500 of dependent credits instead of the $2,200 CTC plus two $500 other-dependent credits, totaling $3,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,899.29 overtime deduction, used a $30,000 rather than $32,200 standard deduction, and added unsupported self-employed health-insurance and half-SE-tax adjustments. Those choices left taxable income at $81,873 instead of $67,794.08."
-us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model omitted the $13,899.29 overtime deduction and then asserted unexplained Medicare or NIIT effects even though neither belongs in this income-tax output and neither applies on these facts. Its $12,500 answer does not follow from its own approximately $9,300 pre-credit regular tax."
-us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied hypothetical TCJA-sunset rules, including a $15,900 standard deduction, personal exemptions, 10%/15% brackets, and a $1,000 CTC. The 2026 computation instead uses the $32,200 MFJ standard deduction, the $13,899.29 overtime deduction, current brackets, and $3,200 of nonrefundable dependent credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored premiums from wages, itemized unsupported medical premiums, and used personal exemptions while omitting the $32,200 standard deduction and $13,899.29 overtime deduction. It also allowed only a $1,000 CTC rather than $3,200 in total nonrefundable dependent credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income far above $67,794.08 and does not identify the deductions used. The correct computation subtracts the $32,200 standard deduction, $510 QBI deduction, and $13,899.29 overtime deduction before applying $3,200 of nonrefundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied pre-TCJA personal exemptions and a reduced standard deduction, omitted the overtime deduction, and added a $2,000 early-distribution penalty that is not part of this regular income-tax output. It also used only a $1,000 CTC instead of $3,200 of nonrefundable dependent credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used personal exemptions and only a $1,000 child credit, while omitting the $13,899.29 overtime deduction and the two $500 other-dependent credits. Those obsolete and incomplete inputs do not produce the traced $67,794.08 taxable income or $3,200 credit subtraction."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model failed to reduce AGI to $67,794.08 using the $32,200 standard deduction, $510 QBI deduction, and $13,899.29 overtime deduction. Applying the stated child and other-dependent credits to the traced $7,639.29 regular tax yields $4,439.29, not $6,215."
-us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model gave no derivation, and its $4,324 result understates the tax remaining after the specified deductions and credits. Tax on $67,794.08 is $7,639.29, and subtracting the $3,200 nonrefundable credits leaves $4,439.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated unspecified medical and retirement amounts as large enough to eliminate liability, but the traced deductions leave $67,794.08 of taxable income. Regular tax is $7,639.29 and the available $3,200 nonrefundable credits do not reduce it to zero."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and credits eliminated all tax without computing them. The actual deductions leave $67,794.08 taxable, and $3,200 of nonrefundable credits reduce $7,639.29 only to $4,439.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used AGI of $114,422 rather than $114,403.37, omitting $18.63 of the traced above-the-line deductions, including the engine-calculated $126.22 IRA amount. It also rounded the overtime deduction from $13,899.29 to $13,899, producing taxable income of $67,813 instead of $67,794.08 and tax $2.27 too high."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model gave no supporting arithmetic, and $2,219 is consistent with subtracting excessive credits or deductions from the regular tax. The applicable deductions yield $7,639.29 before credits, and the nonrefundable credit subtraction is exactly $3,200."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model named the correct deduction categories but did not apply their exact traced amounts and 2026 bracket arithmetic. The exact taxable income is $67,794.08, producing $7,639.29 before the $3,200 credit subtraction and $4,439.29 afterward."
-us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model referred only to the qualifying-child credit and omitted the two $500 credits for the disabled 27-year-old dependent and the 18-year-old dependent. The full nonrefundable reduction is $3,200, applied to $7,639.29 of regular tax."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model relied on gross household income without applying the traced $46,609.29 of below-AGI deductions or the $3,200 nonrefundable credits. Those steps reduce taxable income to $67,794.08 and final pre-refundable-credit tax to $4,439.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied TCJA-sunset personal exemptions, a $17,000 standard deduction, 10%/15% brackets, and an obsolete $110,000 CTC phaseout. The operative computation uses a $32,200 standard deduction, the $13,899.29 overtime deduction, current brackets, and $3,200 of unphased nonrefundable credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored insurance premiums from wages, double-counted the $14,000 self-employment partnership amount, and omitted the $13,899.29 overtime deduction. It also treated $1,796 as QBI instead of using the traced $510 QBI deduction and allowed only the $2,000 child credit rather than $3,200 total credits."
-us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model omitted the $13,899.29 overtime deduction and treated the $1,796 domestic-production input as a direct deduction instead of using the traced $510 QBI deduction. It also added a $2,000 early-withdrawal penalty, which is outside this regular federal-income-tax-before-refundable-credits quantity."
-us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the $14,000 self-employment partnership income on top of the listed $19,350 partnership/S-corp income and deducted retirement contributions belonging to a dependent's separate income calculation. It also omitted the $13,899.29 overtime deduction, overstated QBI at $6,670 instead of $510, and omitted one $500 other-dependent credit."
-us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model failed to reduce wages by the $2,315 pre-tax 401(k) contribution and used only $108 rather than the traced $126.22 IRA deduction, overstating AGI by $2,333.22. It also substituted the $1,796 domestic-production input for the traced $510 QBI deduction, leaving taxable income $1,047.92 too high."
-us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer assumes deductions and credits eliminate liability, but the traced deductions leave $67,794.08 taxable. Tax of $7,639.29 exceeds the $3,200 nonrefundable credits by $4,439.29."
-us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model omitted the $13,899.29 overtime and $510 QBI deductions and incorrectly asserted that dependent credits phase out at this income, despite the MFJ phaseout threshold being far higher. Its stated $7,629.40 pre-credit tax also cannot arithmetically become the submitted $14,629.40."
-us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model simultaneously subtracted a standard deduction and itemized medical deductions, invented $11,827 of retirement contributions and a $12,150 child credit, and mishandled the capital loss. Its final $16,471.63 reverses the direction of its own tax-and-credit arithmetic; the applicable $3,200 credit must be subtracted from $7,639.29."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached nearly the correct taxable income but used approximate 2026 bracket thresholds and tax arithmetic, then reported $5,959 despite its own stated subtraction implying about $4,430. Applying the exact brackets to $67,794.08 produces $7,639.29 before the $3,200 credit."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $14,600 single-filer-sized standard deduction instead of the $32,200 MFJ deduction and omitted the $13,899.29 overtime and $510 QBI deductions. It also invented unspecified additional credits rather than applying the exact $3,200 child and other-dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,899.29 qualified-overtime deduction, leaving taxable income around $80,733 instead of $67,794.08. It also incorrectly split the child tax credit into refundable and nonrefundable portions even though the $7,639.29 pre-credit tax absorbs the full $3,200 nonrefundable credit."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model omitted the qualified-overtime deduction and then improperly folded self-employment tax and unexplained 'additional considerations' into the requested federal income-tax output. This output excludes self-employment tax and equals regular income tax after the $3,200 nonrefundable credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the domestic-production input as an AGI deduction, overstated QBI at roughly $3,900 instead of $510, and omitted the $13,899.29 overtime deduction. Its stated taxable income and tax also do not arithmetically support the reported $11,200 after $2,500 of credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $1,560 self-employed health-insurance deduction and half-SE-tax deduction, used a $30,000 rather than $32,200 standard deduction, and entirely omitted the $13,899.29 overtime deduction. The trace instead produces $67,794.08 of taxable income and subtracts $3,200 of credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,899.29 overtime deduction and incorrectly treated the child tax credit as largely refundable despite sufficient regular tax to use it nonrefundably. It then added unspecified Medicare or NIIT effects even though neither belongs in this regular income-tax result."
+us,scenario_064,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly applied assumed TCJA-sunset rules, including personal exemptions, 10%/15% brackets, and only a $1,000 child credit. The applicable 2026 computation uses the $32,200 standard deduction, the overtime and QBI deductions, current brackets, and $3,200 of nonrefundable dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly excluded employer-sponsored premiums from wages, itemized $21,880 of medical costs not supported by the inputs, and used personal exemptions. The correct computation instead uses $114,403.37 AGI and the standard, overtime, and QBI deductions, followed by $3,200 of credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $11,090.45 implies that the model failed to reduce $114,403.37 AGI by the complete $46,609.29 deduction stack. Those deductions yield $67,794.08 of taxable income, $7,639.29 of regular tax, and $4,439.29 after credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied obsolete personal exemptions and a reduced post-sunset standard deduction, then added a $2,000 early-distribution penalty to an output defined as regular income tax after nonrefundable credits. The requested value excludes that penalty and uses the $32,200 standard and $13,899.29 overtime deductions."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used personal exemptions and only a $1,000 child credit under an obsolete-law calculation. The correct credit is $2,200 for the qualifying child plus two $500 other-dependent credits, against tax calculated from $67,794.08 of taxable income."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The reported $6,215 does not reflect the complete deduction stack that reduces taxable income to $67,794.08. Applying exact 2026 rates and the $3,200 nonrefundable credits yields $4,439.29."
+us,scenario_064,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model understated the final liability by $115.29 while naming the correct credit categories. Exact 2026 rate-bracket arithmetic produces $7,639.29 before the $3,200 credits, not the $7,524 pre-credit amount implied by its answer."
+us,scenario_064,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed that retirement, medical, and dependent provisions eliminated liability. The allowed deductions leave $67,794.08 taxable, and the $3,200 nonrefundable credits reduce $7,639.29 only to $4,439.29."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced liability to zero without computing the allowed deductions or credit limitation. Tax remains because $67,794.08 of taxable income generates $7,639.29 of tax, exceeding the $3,200 nonrefundable credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $114,422 AGI and $67,813 taxable income instead of the trace amounts $114,403.37 and $67,794.08, reflecting incomplete exact treatment of the $2,701.30 pre-tax 401(k) contributions and $126.22 IRA deduction. That $18.92 taxable-income overstatement raises regular tax by $2.27, exactly explaining its $4,441.56 answer."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The $2,219 answer effectively subtracts about twice the allowable nonrefundable dependent credits from the correctly structured tax calculation. Only $3,200 is subtracted from $7,639.29, leaving $4,439.29."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model named the relevant income, deductions, and credits but overstated the result by $45.71 through approximate parameter and bracket arithmetic. Exact taxable income is $67,794.08 and exact pre-credit tax is $7,639.29."
+us,scenario_064,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model referred only to the qualifying-child credit and failed to apply both $500 other-dependent credits for the disabled 27-year-old and the 18-year-old. It also did not use the exact deduction and bracket computation that produces $7,639.29 before the full $3,200 credit."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model's generic high-income shortcut failed to apply the $46,609.29 total deductions and $3,200 nonrefundable credits. Those provisions reduce the requested liability to $4,439.29 despite the household's gross income."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed TCJA-sunset personal exemptions, 10%/15% brackets, a smaller standard deduction, and the old $1,000 CTC phaseout. The applicable calculation uses the $32,200 standard deduction, $13,899.29 overtime deduction, current brackets, and an unphased $3,200 credit."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the overtime deduction in arriving at AGI and then omitted it from its taxable-income deductions, while overstating QBI at $3,310 rather than $510. The overtime deduction belongs below AGI and reduces taxable income to $67,794.08."
+us,scenario_064,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double counted the $14,000 self-employment partnership amount, improperly excluded $21,208 of insurance premiums from wages, and omitted the $13,899.29 overtime deduction. It also applied only the $2,000 child credit rather than the full $3,200 child and other-dependent credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model improperly added the $2,000 early-distribution penalty to the requested regular income-tax output and treated the domestic-production input as a deduction. The requested value excludes that penalty and instead uses the $13,899.29 overtime and $510 QBI deductions."
+us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double counted the $14,000 self-employment partnership amount in addition to the $19,350 partnership/S-corp income and deducted retirement contributions belonging to a dependent. It also omitted the $13,899.29 overtime deduction, overstated QBI, and failed to include the second $500 other-dependent credit."
+us,scenario_064,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used $116,737 AGI because it failed to account for the complete above-the-line loss treatment and exact pre-tax retirement deductions that produce $114,403.37. It also substituted a $1,796 domestic-production deduction for the trace's $510 QBI deduction, leaving taxable income $1,047.92 too high."
+us,scenario_064,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer ignores the positive taxable income remaining after all allowed deductions. The correct deduction stack leaves $67,794.08 taxable and $4,439.29 after nonrefundable credits."
+us,scenario_064,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model double counted the $14,000 self-employment partnership amount, inflating AGI to $127,253, and omitted the $13,899.29 overtime deduction. It also used a $1,796 domestic-production deduction and $3,310 QBI deduction instead of the trace's $510 QBI deduction."
+us,scenario_064,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model's stated $7,629.40 pre-credit tax is close to the correct $7,639.29, but it then declared credits unavailable despite income being far below the $400,000 MFJ phaseout threshold. It also reported $14,629.40, which contradicts its own tax computation; the full $3,200 credit must be subtracted."
+us,scenario_064,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model simultaneously subtracted a standard deduction and invented itemized medical and retirement deductions, used the capital loss twice, and then added a purported $12,150 child credit to tax instead of subtracting the applicable $3,200 credit. Its reported $16,471.63 is arithmetically incompatible with its own stated $4,564.81 regular liability."
 us,scenario_064,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the full $2,000 Child Tax Credit for the age-12 child as refundable. The household's tax liability absorbs that credit nonrefundably, leaving no Additional Child Tax Credit and total federal refundable credits of $0."
 us,scenario_064,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly stated that the household's substantial tax liability absorbs the CTC nonrefundably, then incorrectly submitted the $1,700 ACTC per-child cap as though the cap were an automatic refundable amount. A cap limits an otherwise computed refundable portion; it does not create one when the CTC is fully used against tax liability."
 us,scenario_064,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no parseable value for federal_refundable_credits, violating the required outputs contract."
@@ -3864,28 +4060,30 @@ us,scenario_064,federal_refundable_credits,kimi-k2.6,llm_error,credit_phaseout,F
 us,scenario_064,free_school_meals_eligible,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the negative farm income and capital loss when estimating school-meal countable income. The applicable calculation counts $97,295 of employment income plus $20,000 of retirement distributions, producing $117,295 and a 3.03 FPG ratio, so the household exceeds even the 185% reduced-price threshold and receives no free-meal support."
 us,scenario_064,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_064,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Wisconsin employee payroll tax even though Wisconsin imposes no mandatory employee-side state payroll tax in this calculation. It then replaced its own $10,361.92 component sum with an unsupported $12,318.64 adjustment."
-us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly derived $6,032.29 of Social Security tax and $1,410.78 of Medicare tax, totaling $7,443.07, but submitted $7,407.36 instead. This is a final-answer transcription or arithmetic-contract error."
-us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model improperly deducted the separately listed $1,560 health-insurance premium from FICA wages. The payroll-tax base remains the full $97,295 of wages, producing $7,443.07."
-us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model improperly deducted both the $19,648 employer-sponsored-insurance premium and the $2,315 traditional 401(k) contribution from FICA wages. Neither deduction reduces the engine's $97,295 Social Security and Medicare wage base here."
-us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the entire $19,648 employer-sponsored-insurance premium as a pre-tax FICA deduction and taxed only $77,647. PolicyEngine applies FICA to the full $97,295 of wages."
-us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model subtracted the full $19,648 employer-sponsored-insurance premium from FICA wages. The applicable wage base is $97,295, not $77,647, so Social Security and Medicare total $7,443.07."
-us,scenario_064,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model reduced FICA wages for health-insurance deductions that the engine does not subtract from this payroll-tax base. Applying 6.2% Social Security and 1.45% Medicare rates to the full $97,295 yields $7,443.07."
-us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required structured-output contract."
-us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $12,306 is inconsistent with applying employee Social Security and Medicare rates to the only listed wages of $97,295. It reflects an inflated payroll-tax base despite correctly stating that Wisconsin adds no employee state payroll tax."
-us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model's $8,659 estimate exceeds the 7.65% employee FICA tax on the only listed wages and therefore uses an inflated wage base or adds an inapplicable tax. Wages are below the Additional Medicare Tax threshold, so the total is limited to $7,443.07."
-us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required structured-output contract."
-us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer ignores the head's explicitly listed $97,295 of wages. Those wages generate $6,032.29 of employee Social Security tax and $1,410.78 of employee Medicare tax."
-us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly computed $6,032.29 of Social Security tax and $1,410.78 of Medicare tax with no Additional Medicare Tax, but submitted $8,529.65 instead of their $7,443.07 sum. This is a final aggregation or answer-transfer error."
-us,scenario_064,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model constructed a self-employment tax base by combining the $14,000 self-employment partnership input, $19,350 partnership/S-corp income, and the farm loss, even though the computed person-level taxable net earnings are zero. It also incorrectly treated the deduction for one-half of self-employment tax as a reduction of the tax liability itself; that deduction affects adjusted gross income, not self-employment tax."
-us,scenario_064,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model directly treated the $14,000 self-employment partnership input as taxable net earnings and applied the 92.35% adjustment and 15.3% rate. PolicyEngine’s computed self-employment-tax base is zero for the head and every other person, so there is no amount to which those factors apply."
-us,scenario_064,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly reached a nonpositive result when netting the $14,000 against the farm loss, then abandoned that result and invented a $15,900 positive base by mixing the broader $19,350 partnership/S-corp amount with its $14,000 self-employment subset. The computed person-level net-earnings base is zero, so neither the positive-only treatment nor the double-counted reconstruction is valid."
-us,scenario_064,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model applied the self-employment tax formula directly to the $14,000 self-employment partnership input. It skipped the required computation of the taxable person-level net-earnings base, which is zero for all household members."
-us,scenario_064,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. The required numeric output was $0 because every person’s computed net earnings subject to self-employment tax are zero.
-us,scenario_064,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model asserted a positive self-employment partnership tax base without deriving it from the person-level net-earnings computation. That computation yields zero taxable self-employment earnings for every person, so no standard net-earnings adjustment or self-employment tax rate applies."
-us,scenario_064,self_employment_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model mechanically multiplied the $14,000 input by 92.35% and then by the Social Security and Medicare self-employment rates. It mistook that input for the computed taxable net-earnings base, which is zero across all five people."
-us,scenario_064,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. The required numeric output was $0 because the computation produces no net earnings subject to self-employment tax for any household member.
-us,scenario_064,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model repeatedly derived a zero or negative combined self-employment base after accounting for the farm loss, but discarded that derivation and taxed the $14,000 positive component in isolation. Self-employment tax is based on the computed net earnings rather than grossing up a selected positive component, and that computed base is zero for every person."
+us,scenario_064,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Wisconsin employee payroll tax and then replaced even its own resulting $10,361.92 total with an unsupported $12,318.64 adjustment. Wisconsin contributes no mandatory employee-side state payroll tax here, leaving only $6,032.29 of Social Security and $1,410.78 of Medicare tax."
+us,scenario_064,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model derived the exact components and their correct $7,443.07 sum, then submitted $7,407.36 instead. This is a final-answer transcription or arithmetic inconsistency, not a tax-rule error."
+us,scenario_064,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model incorrectly deducted the separately listed $1,560 health-insurance premiums from Social Security and Medicare wages. FICA applies to the full $97,295 wage input in this case, producing $7,443.07."
+us,scenario_064,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model subtracted both the $19,648 employer-sponsored-insurance premium and the $2,315 traditional 401(k) contribution from FICA wages. Neither reduces the payroll-tax wage base represented by the supplied gross-wage input here, so FICA is calculated on $97,295."
+us,scenario_064,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model treated the $19,648 employer-sponsored-insurance premium as a pre-tax employee deduction and reduced FICA wages to $77,647. The full $97,295 is subject to Social Security and Medicare tax."
+us,scenario_064,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly removed $19,648 of employer-sponsored-insurance premiums from the FICA wage base. Applying 6.2% Social Security and 1.45% Medicare rates to the full $97,295 yields $7,443.07."
+us,scenario_064,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model reduced payroll-taxable wages for health-insurance premiums. The benchmark's $97,295 gross-wage input is fully subject to employee Social Security and Medicare taxes, so the base is not $77,647."
+us,scenario_064,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required output contract."
+us,scenario_064,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The submitted $12,306 is inconsistent with applying ordinary employee Social Security and Medicare rates to the sole $97,295 wage amount, despite the explanation naming that method. The correct employee-side calculation is $97,295 × 6.2% plus $97,295 × 1.45% = $7,443.07, with no Wisconsin employee payroll tax."
+us,scenario_064,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The $8,659 estimate exceeds the $7,443.07 obtained by applying 6.2% Social Security and 1.45% Medicare to the only wages and therefore incorporates an unsupported base or payroll-tax component. Additional Medicare Tax is zero because wages do not cross its threshold."
+us,scenario_064,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax output or explanation, violating the required output contract."
+us,scenario_064,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omitted employee FICA entirely even though the head has $97,295 of wages. Those wages generate $6,032.29 of Social Security tax and $1,410.78 of Medicare tax."
+us,scenario_064,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $2,315 traditional 401(k) contribution from Social Security and Medicare wages. Traditional 401(k) deferrals do not reduce FICA wages, so both taxes apply to the full $97,295."
+us,scenario_064,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly computed $6,032.29 of Social Security tax and $1,410.78 of Medicare tax and correctly found no Additional Medicare Tax, but then submitted $8,529.65 instead of their $7,443.07 sum. The error is an unsupported final aggregation inconsistent with its own stated components."
+us,scenario_064,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model manually combined partnership/S-corporation income, self-employment partnership income, and the farm loss into a $16,550 SECA base even though the entity-level computation contained no net earnings subject to self-employment tax. It also incorrectly treated the income-tax deduction for one-half of self-employment tax as a reduction of the tax liability itself."
+us,scenario_064,self_employment_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model treated the $14,000 self-employment partnership input as a standalone positive SECA base while disregarding its own conclusion that the farm loss eliminates positive net self-employment earnings. The person-level taxable net-earnings computation is zero, so applying 92.35% and 15.3% to $14,000 creates a nonexistent liability."
+us,scenario_064,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly reached a nonpositive result after combining the $14,000 partnership amount with the farm loss, then abandoned that result and invented a $15,900 base by adding the broader $19,350 partnership/S-corporation amount. That double counts partnership-related income and overrides the zero entity-level net-earnings base."
+us,scenario_064,self_employment_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model directly applied the self-employment tax formula to the $14,000 partnership-related input. It skipped the entity-level determination of net earnings subject to SECA, which yields zero for every household member."
+us,scenario_064,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. It therefore failed the required structured-output contract before any substantive tax computation could be evaluated.
+us,scenario_064,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,The model assumed the positive self-employment partnership input itself was the taxable net-earnings base and applied a standard adjustment to it. The required person-level SECA-base computation yields zero rather than a positive adjusted amount.
+us,scenario_064,self_employment_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model calculated both SECA components on 92.35% of $14,000 without first establishing positive net earnings subject to self-employment tax. The household has a zero person-level SECA base, so neither the Social Security nor Medicare self-employment component applies."
+us,scenario_064,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable self_employment_tax output. It therefore failed the required structured-output contract before any substantive tax computation could be evaluated.
+us,scenario_064,self_employment_tax,ox-alpha,llm_error,payroll_tax_base,False,"The model constructed a $16,550 self-employment base by adding the full $19,350 partnership/S-corporation amount and the $14,000 self-employment partnership amount before subtracting the farm loss. This double counts overlapping partnership-related fields and replaces the zero entity-level taxable net-earnings result with a fabricated positive base."
+us,scenario_064,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model repeatedly derived a zero or negative combined self-employment result, then discarded it and taxed the $14,000 positive partnership-related line in isolation. Self-employment tax is imposed on net earnings after the applicable activity-level computation, not on gross positive components separated from offsetting results; here that person-level base is zero."
 us,scenario_064,snap,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_064,spouse_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_064,ssi,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated Dependent 1's generic disability flag as satisfying SSI's program-specific aged, blind, or disabled criterion. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so applying the one-third in-kind-support reduction and a federal benefit rate was invalid; SSI is $0 before those calculations arise."
@@ -3897,36 +4095,38 @@ us,scenario_064,ssi,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The mod
 us,scenario_064,ssi,inkling,llm_error,categorical_eligibility,False,"The model incorrectly admitted Dependent 1 into SSI based on the generic disability flag, then applied a one-third living-arrangement reduction and an estimated Wisconsin supplement. Dependent 1 has `is_ssi_aged_blind_disabled=False`, so neither the living-arrangement adjustment nor the supplement is reached."
 us,scenario_064,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
 us,scenario_064,ssi,kimi-k3,llm_error,categorical_eligibility,False,"The model declared Dependent 1 categorically SSI-eligible from the generic disability fact and then applied the full individual federal rate after excluding parental deeming. Dependent 1 instead fails `is_ssi_aged_blind_disabled`, so deeming, resources, and the benefit rate never enter the computation."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own bracket calculation produced $4,964, but it then asserted an unsupported reduction to $2,699 for unspecified capital-loss, farm, and deduction adjustments. Those adjustments were already reflected in the income computation and do not create a $2,265 tax reduction."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an approximate $104,434 taxable-income base assembled directly from gross income components instead of the traced Wisconsin taxable income of $102,144.25. It also described the state calculation as starting from federal taxable income rather than applying Wisconsin deductions and exemptions to AGI."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"After estimating gross Wisconsin tax near $5,026, the model subtracted about $875 of school-property and itemized-deduction credits without listed property tax or a computed qualifying itemized-deduction credit. That invented credit reduction drove the answer below the traced liability."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed bracket tax of $4,784 on an approximate $100,000 base and then returned $5,074 despite describing exemptions and deductions as reductions. Its final number contradicts its own arithmetic and fails to apply the traced $102,144.25 taxable-income base consistently."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model rounded a bracket computation to $4,600 instead of calculating the exact tax on $102,144.25. Applying the 2026 Wisconsin schedule to that traced taxable income yields $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model reduced the Wisconsin standard deduction to $3,957 using an invented linear phaseout, rather than the traced $8,759.12 deduction. It then subtracted an unsupported $520 of credits, so both its taxable income and post-credit liability were wrong."
-us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double counted the $14,000 self-employment partnership amount in addition to the listed $19,350 partnership/S-corporation income and treated the $1,796 domestic-production deduction as an AGI subtraction. It also counted only four exemptions instead of five, corrupting the Wisconsin taxable-income base."
-us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a flat $25,000 Wisconsin standard deduction and subtracted the domestic-production deduction from AGI. The applicable income-dependent deduction is $8,759.12, producing taxable income of $102,144.25 rather than $82,566."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a $13,506 standard deduction instead of $8,759.12 and therefore understated taxable income by $4,746.88. It also subtracted a $150 dependent credit that is absent from the traced pre-refundable-credit computation."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained estimate did not execute the exact Wisconsin calculation: $114,403.37 minus $8,759.12 and $3,500 gives $102,144.25, whose bracket tax is $4,606. Its $4,652.12 answer reflects an imprecise bracket or taxable-income estimate."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model started from AGI of $92,978 instead of $114,403.37 and consequently used only about $78,969 of taxable income. The omitted taxable income caused the substantial understatement."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used net taxable income of $82,256 instead of $102,144.25, overstating Wisconsin deductions by $19,888.25. The traced deductions are limited to the $8,759.12 standard deduction and $3,500 of personal exemptions."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The answer supplies no computation and understates the tax by $481. The required bracket calculation uses Wisconsin taxable income of $102,144.25 and yields $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The answer supplies no deduction or credit calculation supporting its $3,212 figure. The listed Wisconsin standard deduction and five exemptions leave $102,144.25 taxable, whose 2026 bracket tax is $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model omitted the requested output entirely, so no Wisconsin tax value was available for evaluation."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly treated deductions and unspecified household credits as sufficient to eliminate all Wisconsin tax. They reduce AGI only to $102,144.25 of taxable income, and no traced nonrefundable credit erases the resulting $4,606 liability."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model assumed unspecified deductions and credits exceeded Wisconsin gross tax without calculating either. The actual deductions leave $102,144.25 taxable and the applicable brackets produce $4,606 before refundable credits."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $15,166 standard deduction instead of $8,759.12 and counted only two $700 exemptions instead of five. It then invented a $472 itemized-medical credit, further understating the liability."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The unsupported $2,800 estimate does not follow the traced Wisconsin computation. AGI of $114,403.37 less $12,259.12 of deductions and exemptions leaves $102,144.25 taxable and $4,606 of tax."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the correct general method but returned $4,872 rather than applying the 2026 joint brackets precisely to $102,144.25. The exact bracket computation yields $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The model gave only a rounded estimate and did not calculate the Wisconsin income base or brackets. The traced taxable income is $102,144.25, producing $4,606 rather than $4,071."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model rounded the state tax to $4,500 without calculating the deduction, exemptions, or brackets. Taxing the traced $102,144.25 under the 2026 Wisconsin schedule gives $4,606."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction of about $6,500 and taxable income near $107,900. The traced deduction is $8,759.12 and five exemptions subtract another $3,500, leaving $102,144.25 taxable."
-us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied 2024 Wisconsin rates of 4.75% and 5.75% rather than the 2026 schedule used by the benchmark. It also started from the wrong AGI and taxable-income base, so its $4,760 calculation does not represent the requested tax year."
-us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model started from AGI near $97,723 and treated the listed overtime premium as a separate Wisconsin addback. The overtime premium is already included in annual wages, while the traced Wisconsin AGI is $114,403.37 and taxable income is $102,144.25."
-us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested output entirely, so no Wisconsin tax value was available for evaluation."
-us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied a 6.27% marginal rate to income above its second bracket, but the applicable 2026 rate for this portion of joint taxable income is 5.3%. That wrong rate inflated the liability to $6,091."
-us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero without any calculation. Wisconsin deductions and exemptions leave $102,144.25 taxable, so the graduated schedule produces $4,606 rather than eliminating the liability."
-us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly removed the $13,899 overtime premium from wages even though gross wages are annual totals that already include overtime and the benchmark does not specify a Wisconsin wage exclusion. It also used incorrect bracket boundaries and a 4.65% rate, producing the wrong tax base and schedule."
-us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model subtracted $11,827 of retirement contributions and medical expenses directly from Wisconsin AGI even though those amounts are not traced Wisconsin income deductions here. The correct reductions are the $8,759.12 standard deduction and $3,500 of exemptions, leaving $102,144.25 taxable."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model reached a pre-credit bracket tax near $4,964 but then subtracted unsupported nonrefundable credits to force the result down to $2,699. The traced calculation uses AGI of $114,403.37, an $8,759.12 standard deduction, and $3,500 of exemptions, with no such $2,265 credit reduction."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an approximate $104,434 tax base assembled directly from gross income components and deductions instead of the traced Wisconsin taxable income of $102,144.25. Applying approximate rates to that incorrect base produced $4,512 rather than the bracket result of $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an approximate $9,000 standard deduction, counted only four exemptions, and then invented roughly $875 of school-property and itemized-deduction credits. Wisconsin instead subtracts an $8,759.12 standard deduction and five exemptions totaling $3,500 from $114,403.37, yielding $102,144.25 and $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model estimated Wisconsin AGI near $113,000 and taxable income near $100,000 rather than using $114,403.37 and $102,144.25. It also described personal exemptions as a credit-equivalent adjustment and then reported $5,074, a result inconsistent with its own $4,784 bracket calculation."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model rounded the bracket calculation to $4,600 instead of applying the 2026 Wisconsin schedule exactly to traced taxable income of $102,144.25. That exact computation produces $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at $116,040 and reduced the sliding standard deduction to $3,957, producing taxable income of $108,583 instead of $102,144.25. It then subtracted an invented $520 of unspecified credits; the traced inputs are AGI $114,403.37, standard deduction $8,759.12, and exemptions $3,500."
+us,scenario_064,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted $14,000 of self-employment partnership income alongside the listed $19,350 partnership/S-corporation income and treated the $1,796 domestic-production deduction as an AGI subtraction. It consequently started from $126,626 rather than Wisconsin AGI of $114,403.37 and never computed the traced $102,144.25 taxable income."
+us,scenario_064,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the $1,796 domestic-production deduction from its AGI and used a flat $25,000 Wisconsin standard deduction. The applicable income-dependent deduction is $8,759.12, leaving taxable income of $102,144.25 rather than $82,566."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a $13,506 standard deduction instead of $8,759.12 and therefore understated taxable income at $97,576 rather than $102,144.25. It also subtracted an unsupported $150 dependent credit from the bracket tax."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $4,652.12 estimate does not apply the exact 2026 Wisconsin joint-filer schedule to traced taxable income of $102,144.25. That bracket computation yields $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin AGI at $92,978 instead of $114,403.37 and consequently used only about $78,969 of taxable income. After the $8,759.12 standard deduction and $3,500 exemptions, the correct tax base is $102,144.25."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin taxable income at $82,256 by misapplying the sliding standard deduction. The traced deduction is $8,759.12 from AGI of $114,403.37, followed by $3,500 of exemptions, leaving $102,144.25."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no computation supporting $4,125 and failed to apply the traced Wisconsin calculation. Taxing $102,144.25 under the 2026 joint-filer brackets produces $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $3,212 answer reflects excessive deductions or unsupported nonrefundable credits. Wisconsin permits an $8,759.12 standard deduction and $3,500 of exemptions here, leaving $102,144.25 to generate $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions, retirement adjustments, and household circumstances as sufficient to erase Wisconsin taxable income. They reduce $114,403.37 only to $102,144.25, which produces $4,606 of tax before refundable credits."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed deductions and credits exceed the Wisconsin tax without calculating them. The allowed $8,759.12 standard deduction and $3,500 exemptions leave $102,144.25 taxable, and no traced nonrefundable credit reduces the resulting $4,606 to zero."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an inflated $15,166 standard deduction, counted only two $700 exemptions instead of five, and subtracted an unsupported $472 medical/itemized-deduction credit. The traced deductions are $8,759.12 plus $3,500, yielding taxable income of $102,144.25 and tax of $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The unexplained $2,800 estimate embodies excessive Wisconsin deductions, exemptions, or nonrefundable credits. The traced deductions total $12,259.12 from AGI of $114,403.37, leaving $102,144.25 and $4,606 of tax."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model named the correct broad method but did not use the exact Wisconsin deduction and bracket parameters. The $8,759.12 standard deduction and $3,500 exemptions create a $102,144.25 tax base whose 2026 joint-schedule liability is $4,606, not $4,872."
+us,scenario_064,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's unsupported $4,071 estimate does not reflect the traced taxable-income calculation. Wisconsin AGI of $114,403.37 less $8,759.12 and $3,500 leaves $102,144.25, producing $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the liability to $4,500 without identifying the tax base or applying the exact bracket schedule. The traced taxable income is $102,144.25, and the 2026 Wisconsin joint-filer calculation yields $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model reduced the Wisconsin standard deduction to about $6,500 and thereby overstated taxable income near $107,900. The traced deduction is $8,759.12 and five exemptions subtract another $3,500, leaving $102,144.25."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model came close to the traced tax base but used incorrect bracket widths of $29,350 for the first two bands. Applying the actual 2026 Wisconsin joint-filer brackets to $102,144.25 yields $4,606, not $4,610."
+us,scenario_064,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used obsolete 2024 bracket rates of 4.75% and 5.75% instead of the applicable 2026 rates and also started from an incorrect $106,225 AGI. The traced 2026 computation taxes $102,144.25 and produces $4,606."
+us,scenario_064,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model started from an understated federal AGI near $97,723 and treated the $13,899 overtime amount as a Wisconsin addback, arriving at taxable income near $90,000. The traced Wisconsin AGI is $114,403.37 and taxable income after the state deduction and exemptions is $102,144.25."
+us,scenario_064,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
+us,scenario_064,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model applied a nonexistent 6.27% Wisconsin marginal band to this income instead of the applicable 5.3% band and understated the standard deduction at $5,481. The traced $8,759.12 deduction produces $102,144.25 of taxable income and $4,606 of tax."
+us,scenario_064,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model returned zero without a substantive calculation. Wisconsin AGI of $114,403.37 remains $102,144.25 after the allowed standard deduction and exemptions, so the liability is not eliminated."
+us,scenario_064,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at $127,253 but then used an inapplicable $31,300 standard deduction and incorrect bracket thresholds. Wisconsin instead uses AGI of $114,403.37, an $8,759.12 standard deduction, and $3,500 of exemptions, leaving $102,144.25."
+us,scenario_064,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model improperly excluded the $13,899 FLSA overtime premium from AGI and used approximate, incorrect Wisconsin bracket rates and thresholds. The traced Wisconsin AGI is $114,403.37, not $100,523, and the resulting taxable income is $102,144.25."
+us,scenario_064,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $11,827 of retirement contributions and deductible medical expenses directly in constructing Wisconsin taxable income. The traced calculation subtracts only the $8,759.12 Wisconsin standard deduction and $3,500 of exemptions from $114,403.37, yielding $102,144.25 and $4,606."
 us,scenario_064,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly ruled out every Wisconsin refundable-credit pathway it identified, including the homestead credit, Wisconsin earned income credit, farmland preservation credit, and veterans property tax credit, but then inserted an unexplained $154 residual. With every applicable component equal to zero, the required aggregation yields $0."
 us,scenario_064,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract."
 us,scenario_064,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract."
@@ -3947,78 +4147,82 @@ us,scenario_066,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligib
 us,scenario_066,head_medicaid_eligible,grok-4.3,llm_error,health_coverage,False,"The model incorrectly made other means-tested health coverage preclusive. It is not an exclusion from the Virginia ACA adult expansion category, under which the head qualifies based on age, status, and MAGI."
 us,scenario_066,head_medicaid_eligible,minimax-m3,llm_error,asset_resource,False,"The model applied a $2,000 asset limit from non-MAGI Medicaid pathways to Virginia's MAGI-based ACA adult expansion category. That category has no asset test, so the $14,000 bank balance does not offset eligibility at 0.03 times FPL."
 us,scenario_066,head_medicaid_eligible,qwen-3.7-max,llm_error,period_annualization,False,"The model improperly annualized $32 per hour at 40 hours per week into $66,560 despite the prompt explicitly defining $520 as the full-year gross wage total. It also wrongly treated other means-tested health coverage as disqualifying; using the stated annual income places the head at 0.03 times FPL and within Virginia's adult expansion category."
-us,scenario_066,snap,claude-fable-5,llm_error,period_annualization,False,The model held the $298 monthly maximum constant for all 12 months. It omitted the later-month increase to $304.68 and therefore missed the annual aggregation of month-specific allotments.
-us,scenario_066,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary SNAP resource limit to the $14,000 bank balance. The household is categorically eligible through TANF non-cash assistance, so that balance does not cause SNAP ineligibility."
-us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,The model substituted an estimated $275 monthly maximum for the applicable 2026 monthly allotments. The trace uses $298 in most months and $304.68 in later months.
-us,scenario_066,snap,claude-opus-4.8,llm_error,other,False,"The model correctly reached essentially zero net income and identified a maximum-allotment case, but then reduced the result to $2,496 without a valid SNAP computation. Zero net income produces the full month-specific maximum allotment, totaling $3,596.04."
-us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,The model used an obsolete or estimated $283 monthly maximum. The applicable allotments are $298 in most months and $304.68 in later months.
-us,scenario_066,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly denied Virginia categorical eligibility and imposed the ordinary resource ceiling. TANF non-cash assistance establishes categorical eligibility, so the $14,000 bank balance does not disqualify the household."
-us,scenario_066,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model misread the explicitly annual $520 wage amount as $520 per week. Correct monthly gross income is $43.33, which passes the SNAP income test."
-us,scenario_066,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model imposed the three-month ABAWD time limit despite the listed 40 usual weekly work hours. Forty hours per week satisfies the work requirement, so benefits are calculated for all 12 months."
-us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model used an estimated $291 monthly maximum for every month. The applicable values are $298 in most months and $304.68 after the midyear parameter change.
-us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer ignores that $43.33 of monthly income passes the tests and TANF non-cash assistance supplies categorical eligibility despite the assets. With zero net income, the household receives the full monthly allotments."
-us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly recognized categorical eligibility but used $291 as the maximum in every month. It missed the applicable $298 allotment and its later increase to $304.68.
-us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,The model correctly reached a maximum-benefit case but selected the wrong $291 monthly maximum. The 2026 calculation uses $298 in most months and $304.68 in later months.
-us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,The model treated $292 as the applicable maximum for the entire year. PolicyEngine instead uses $298 in most months and $304.68 in later months.
-us,scenario_066,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,The model used a flat $292 monthly allotment. It omitted both the applicable $298 level and the later-month increase to $304.68.
-us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,The model rounded the maximum allotment to $300 and annualized that approximation. The calculation requires the exact month-specific values of $298 and $304.68.
-us,scenario_066,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the $14,000 account as disqualifying under the ordinary resource rule. TANF non-cash categorical eligibility removes that barrier, while the household's very low income passes the remaining tests."
-us,scenario_066,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned zero without applying the stated take-up assumption or the categorical-eligibility pathway. TANF non-cash assistance establishes categorical eligibility, and zero net income yields positive maximum allotments in every month."
-us,scenario_066,snap,gpt-5.5,llm_error,period_annualization,False,The model multiplied $298 by 12 as though the monthly parameter never changed. It missed the later months calculated at $304.68.
-us,scenario_066,snap,gpt-5.6-luna,llm_error,period_annualization,False,The model held the $298 maximum constant throughout 2026. The annual total must include later-month allotments of $304.68.
-us,scenario_066,snap,gpt-5.6-sol,llm_error,period_annualization,False,The model correctly derived zero net income but annualized a single $298 monthly amount. It omitted the later-month increase to $304.68.
-us,scenario_066,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the noncategorical resource limit to the bank account. TANF non-cash assistance makes the household categorically eligible, so the $14,000 balance does not bar SNAP."
-us,scenario_066,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model denied SNAP under a generic asset limit and treated other health coverage as relevant to that denial. TANF non-cash categorical eligibility prevents the bank balance from disqualifying the household, and health coverage is not a SNAP bar."
-us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly applied categorical eligibility and zero net income but used $292 per month. The applicable monthly allotments are $298 and, later in the year, $304.68."
-us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model selected a flat $292 maximum rather than the month-specific 2026 allotments. It also described the standard deduction as $204 annually instead of a monthly deduction, although the deduction still reduces adjusted income to zero here."
-us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,The model rounded the monthly maximum to about $300 and multiplied by 12. Exact monthly calculation uses $298 in most months and $304.68 in later months.
-us,scenario_066,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model asserted that no categorical waiver was available and imposed the standard resource limit. The household qualifies categorically through TANF non-cash assistance, so its $14,000 bank balance does not eliminate SNAP."
-us,scenario_066,snap,kimi-k3,llm_error,period_annualization,False,The model correctly found categorical eligibility and zero net income but used $298 for all 12 months. It failed to incorporate the later-month allotment of $304.68.
-us,scenario_066,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model applied the ordinary asset ceiling and treated $14,000 as disqualifying. TANF non-cash categorical eligibility bypasses that resource test."
-us,scenario_066,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted an unsupported $872 annual income contribution after acknowledging only $520 of annual earnings. The 20% earned-income deduction and standard deduction reduce SNAP net income to zero, so no 30% contribution is subtracted from the maximum allotment."
-us,scenario_066,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model applied the 30% contribution to gross annual earnings instead of monthly SNAP net income after deductions, and it invented a $149 monthly minimum-benefit floor. The deductions reduce net income to zero, so the household receives the full applicable monthly maximums rather than a minimum benefit."
-us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 15% Virginia refundable-EITC percentage instead of the 2026 rate of 20%. Applying 20% to the unrounded $39.78 federal EITC yields $7.96, not $6.00."
-us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly stated that Virginia has no refundable individual EITC in 2026. This eligible resident can claim the refundable Virginia EITC equal to 20% of the $39.78 federal credit, or $7.96."
-us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model invented a minimum-earnings barrier and treated $520 of wages as producing essentially no childless federal EITC. The federal EITC phases in from the first dollar of earned income and equals $39.78 here, supporting a $7.96 Virginia refundable EITC."
-us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model wrongly rounded a positive credit to zero and used a 15% Virginia percentage. The applicable 20% rate applied to the $39.78 federal EITC produces $7.956, which rounds to $7.96."
-us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model conflated Virginia's nonrefundable low-income credit or EITC option with the refundable EITC pathway. The household qualifies for the refundable option, which is not limited by its zero state tax liability and pays $7.96."
-us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly imposed an approximately $600 minimum earned-income threshold for the childless federal EITC. At $520 of wages the federal EITC is $39.78, and Virginia refunds 20% of it, yielding $7.96."
-us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model failed to apply the prompt's instruction to assume filing and program take-up when required, instead denying the refundable EITC because no election was explicitly stated. The assumed claim activates Virginia's refundable EITC option and produces $7.96."
-us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated Virginia's earned-income credit as exclusively nonrefundable. Virginia provides a refundable EITC pathway for this household, equal to 20% of its $39.78 federal EITC, or $7.96."
-us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model correctly identified the federal EITC base but applied a 15% Virginia percentage instead of 20%. Multiplying $39.78 by 20% and rounding to cents gives $7.96.
-us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Virginia's refundable EITC without evaluating the household's federal EITC eligibility. The household has a $39.78 federal EITC, and the 20% Virginia refundable credit is $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an outdated 15% Virginia refundable-EITC percentage and rounded the federal credit before applying it. The 2026 calculation uses 20% of $39.78, which rounds to $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied 15% rather than the applicable 20% Virginia refundable-EITC rate. The correct multiplication is $39.78 × 0.20 = $7.956, rounded to $7.96."
-us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,The model applied a 15% Virginia refundable-EITC percentage instead of 20%. The $39.78 federal EITC therefore generates $7.96 after rounding.
-us,scenario_066,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the wrong Virginia refundable-EITC rate of 15%. At the applicable 20% rate, the $39.78 federal credit produces $7.96."
-us,scenario_066,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly made refundable-credit availability depend on having state income-tax liability. Virginia's refundable EITC can exceed liability, and this household receives $7.96 based on its $39.78 federal EITC."
-us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model omitted the Virginia refundable EITC triggered by the household's federal EITC eligibility. The $39.78 federal credit generates a $7.96 refundable Virginia credit at 20%.
-us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed away the state refundable credit based on the income level. Low earned income instead produces a $39.78 federal EITC, which qualifies the household for a $7.96 Virginia refundable EITC."
-us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model used a 15% Virginia refundable-EITC percentage rather than the applicable 20% rate. Applying 20% to the unrounded $39.78 federal EITC yields $7.96.
-us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly excluded a childless single adult from Virginia's refundable EITC. This 40-year-old qualifies through the childless federal EITC, and Virginia refunds 20% of the $39.78 federal amount, or $7.96."
-us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the absence of qualifying children as disqualifying for Virginia's refundable credit. The head qualifies for the childless federal EITC of $39.78, producing a $7.96 Virginia refundable EITC."
-us,scenario_066,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model incorrectly conditioned refundable credits on positive state tax liability. Virginia's refundable EITC is payable beyond liability and equals $7.96 for this household.
-us,scenario_066,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model omitted the refundable Virginia EITC available to this low-income single adult. The household's $39.78 federal EITC supports a 20% state credit of $7.96.
-us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Virginia has no applicable refundable state EITC. The household qualifies for Virginia's refundable EITC, calculated as 20% of $39.78 and rounded to $7.96."
-us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model applied a 15% Virginia refundable-EITC percentage instead of 20% and rounded the federal base prematurely. Using 20% of $39.78 yields $7.96.
-us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model incorrectly used zero state tax liability to deny a refundable credit. This household qualifies for a $7.96 Virginia refundable EITC regardless of having no pre-credit state liability.
-us,scenario_066,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model incorrectly treated Virginia's refundable EITC as expiring after tax year 2025. The refundable provision applies in 2026, and 20% of the household's $39.78 federal EITC yields $7.96."
-us,scenario_066,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model omitted the applicable Virginia refundable EITC. Federal EITC eligibility supplies a $39.78 base, and Virginia's 20% refundable credit equals $7.96 after rounding."
-us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model wrongly required positive tax liability or qualifying children for Virginia's refundable EITC. The childless head qualifies for a $39.78 federal EITC, and the refundable Virginia amount is $7.96 even with zero state liability."
-us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated a single-person household with no qualifying children as ineligible for Virginia's refundable EITC. The 40-year-old head qualifies for the childless federal EITC of $39.78, producing a $7.96 state refundable credit."
-us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the parents’ wages and pension in the dependent’s Medicaid MAGI and then evaluated a three-person household against 138% FPL. The dependent’s engine-calculated MAGI income level is 0.00 times FPL, qualifying them through Indiana’s ACA adult expansion category; parental resources and SSI-related rules do not control that pathway."
-us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model treated membership in the parents’ tax unit as sufficient to include their income in the dependent’s Medicaid MAGI. The applicable adult-expansion calculation gives the dependent a MAGI income level of 0.00 times FPL, below Indiana’s 138% limit."
-us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model substituted total household income for the dependent’s Medicaid MAGI. The dependent’s MAGI income level is 0.00 times FPL, so the 23-year-old qualifies under Indiana’s ACA adult expansion pathway."
-us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model incorrectly attributed the parents’ combined income to the dependent merely because the dependent belongs to their tax household. PolicyEngine’s applicant-level MAGI calculation is 0.00 times FPL, which satisfies Indiana’s adult expansion limit."
-us,scenario_067,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used the parents’ $87,302 of wages and pension as the dependent’s countable Medicaid income. The adult-expansion test instead yields a MAGI income level of 0.00 times FPL for the dependent, below 138% FPL."
-us,scenario_067,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model pooled the parents’ earnings and pension with the dependent’s disability benefits and tested the resulting $98,102 against a three-person expansion threshold. The relevant MAGI calculation for the dependent is 0.00 times FPL, making them eligible through Indiana’s ACA adult expansion category."
-us,scenario_067,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model declined to apply Medicaid eligibility because SSI receipt or separate Medicaid-qualification details were not listed. SSI is unnecessary here: age 23 and MAGI income at 0.00 times FPL place the dependent directly in Indiana’s ACA adult expansion category.
-us,scenario_067,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,The model wrongly counted the parents’ income in the dependent’s expansion MAGI and treated SSI or another categorical pathway as necessary. The dependent qualifies directly under Indiana’s adult expansion category because their MAGI income level is 0.00 times FPL.
-us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model applied total household income to the dependent instead of the engine’s Medicaid MAGI calculation for that applicant. The dependent’s MAGI income level is 0.00 times FPL, below Indiana’s 138% adult-expansion threshold."
-us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,asset_resource,False,"The model treated household income and listed resources as disqualifying across all Medicaid pathways. Indiana’s MAGI-based ACA adult expansion pathway has no asset test, and the dependent’s relevant MAGI income level is 0.00 times FPL."
-us,scenario_067,dependent1_medicaid_eligible,grok-4.3,llm_error,asset_resource,False,"The model applied the dependent’s disability benefits and household resources as though they controlled the ACA adult expansion test. The applicable pathway is MAGI-based with no resource test, and PolicyEngine calculates the dependent’s MAGI income level as 0.00 times FPL."
-us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model falsely characterized Indiana as a non-expansion state and analyzed a disabled-adult pathway using parental income. Indiana has adopted ACA Medicaid expansion, and the 23-year-old qualifies in its adult category because their MAGI income level is 0.00 times FPL."
-us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the $10,800 of disability benefits as countable MAGI that exceeds Indiana’s adult Medicaid limit. The engine’s MAGI computation for the dependent is zero, placing them at 0.00 times FPL and within the ACA adult expansion threshold."
+us,scenario_066,snap,claude-fable-5,llm_error,period_annualization,False,"The model held the $298 monthly allotment constant for all 12 months. It omitted the later-2026 increase to $304.68, so its annual aggregation stopped at $3,576 instead of $3,596.04."
+us,scenario_066,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary SNAP resource limit to the $14,000 bank balance. The household qualifies categorically through TANF non-cash assistance, so that balance does not cause ineligibility and the monthly allotments must be calculated."
+us,scenario_066,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated one-person maximum allotment of $275 per month. PolicyEngine applies $298 in most months and $304.68 in later months, totaling $3,596.04."
+us,scenario_066,snap,claude-opus-4.8,llm_error,other,False,"The model correctly concluded that net income is zero and identified a $298 monthly maximum, but then asserted an unexplained reduction to $2,496. With zero SNAP net income there is no 30% income offset; the applicable monthly maximums total $3,596.04."
+us,scenario_066,snap,claude-opus-5,llm_error,thresholds_rates,False,The model substituted a $283 estimated monthly maximum for the applicable 2026 allotments. The correct monthly schedule uses $298 in most months and $304.68 later in the year.
+us,scenario_066,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model treated the household as subject to the ordinary resource test and denied SNAP because of the $14,000 balance. PolicyEngine grants categorical eligibility through TANF non-cash assistance, which bypasses that disqualification."
+us,scenario_066,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model misread the explicitly annual $520 wage amount as $520 per week. The correct monthly gross income is $43.33, which passes the income test and leads to positive SNAP benefits."
+us,scenario_066,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model imposed the three-month ABAWD time limit after declaring that the head failed the work requirement. The stated 40 usual weekly hours satisfies the 20-hour work standard, and the trace confirms the work-requirement test is met for all 12 months."
+us,scenario_066,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an estimated $291 monthly maximum throughout the year. The applicable monthly amounts are $298 in most months and $304.68 in later months, summing to $3,596.04."
+us,scenario_066,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer discards the household despite gross income of $43.33 per month and categorical eligibility through TANF non-cash assistance. Those rules establish eligibility, after which the monthly allotments sum to $3,596.04."
+us,scenario_066,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model correctly applied categorical eligibility but used $291 as the monthly maximum for every month. PolicyEngine uses $298 in most months and $304.68 later in 2026.
+us,scenario_066,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly recognized zero net income and the categorical-eligibility waiver, but selected a $291 monthly maximum. The 2026 monthly schedule instead totals $3,596.04."
+us,scenario_066,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a flat annual maximum of $3,504, equivalent to $292 every month. It missed both the $298 amount used in most months and the later increase to $304.68."
+us,scenario_066,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model assigned a flat $292 monthly maximum. The applicable monthly allotments are higher and change during the year, producing $3,596.04."
+us,scenario_066,snap,glm-5.2,llm_error,thresholds_rates,False,"The model rounded the one-person maximum to $300 and multiplied that estimate by 12. The engine uses the exact monthly values of $298 and later $304.68, whose annual sum is $3,596.04."
+us,scenario_066,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the $14,000 bank balance as disqualifying under the ordinary resource limit. Categorical eligibility through TANF non-cash assistance makes the household eligible despite that balance."
+us,scenario_066,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model returned zero without applying the established eligibility pathway. Gross income is only $43.33 monthly, and TANF non-cash categorical eligibility allows the household to receive allotments totaling $3,596.04."
+us,scenario_066,snap,gpt-5.5,llm_error,period_annualization,False,"The model applied $298 uniformly for 12 months. It omitted the later months at $304.68, understating the annual total by $20.04."
+us,scenario_066,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model froze the monthly maximum at $298 for the full year. The later-2026 monthly allotment rises to $304.68, so the month-by-month sum is $3,596.04."
+us,scenario_066,snap,gpt-5.6-sol,llm_error,period_annualization,False,The model correctly reduced SNAP net income to zero but annualized a constant $298 monthly allotment. It failed to incorporate the later monthly amount of $304.68.
+us,scenario_066,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the noncategorical resource limit to the $14,000 account. The household qualifies categorically through TANF non-cash assistance, so the resource balance does not eliminate SNAP eligibility."
+us,scenario_066,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model denied SNAP under a generic asset limit and treated other health coverage as relevant to SNAP eligibility. TANF non-cash categorical eligibility overrides the resource barrier, and health coverage does not cancel the SNAP allotment."
+us,scenario_066,snap,grok-4.5,llm_error,thresholds_rates,False,The model correctly applied BBCE and zero net income but used $292 as the one-person maximum in every month. The applicable amounts are $298 in most months and $304.68 later in 2026.
+us,scenario_066,snap,grok-4.6,llm_error,thresholds_rates,False,"The model selected a flat $292 monthly allotment after correctly finding zero net income. Using the exact 2026 monthly amounts yields $3,596.04."
+us,scenario_066,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the wrong $292 monthly maximum and therefore computed $3,504. Its reference to a $204 annual standard deduction also confuses the deduction's monthly periodicity, though net income remains zero here."
+us,scenario_066,snap,inkling,llm_error,thresholds_rates,False,"The model rounded the monthly maximum to about $300 and annualized that approximation. The exact month-specific allotments are required and sum to $3,596.04."
+us,scenario_066,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model asserted that no categorical-eligibility waiver was present and applied the standard resource ceiling. PolicyEngine establishes categorical eligibility through TANF non-cash assistance, so the $14,000 balance is not disqualifying."
+us,scenario_066,snap,kimi-k3,llm_error,period_annualization,False,The model correctly found categorical eligibility and zero net income but held the $298 allotment constant across the year. It omitted the later months at $304.68.
+us,scenario_066,snap,minimax-m3,llm_error,categorical_eligibility,False,The model denied eligibility solely because assets exceeded the ordinary SNAP limit. TANF non-cash categorical eligibility prevents that resource test from barring this household.
+us,scenario_066,snap,ox-alpha,llm_error,period_annualization,False,"The model correctly applied categorical eligibility and zero net income, then multiplied $298 by all 12 months. The later monthly allotment is $304.68, making the annual total $3,596.04."
+us,scenario_066,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted an $872 annual income contribution after acknowledging minimal income, instead of calculating SNAP net income after the 20% earned-income and standard deductions. Those deductions reduce net income to zero, so no 30% contribution is subtracted from the applicable maximum allotments."
+us,scenario_066,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model applied the 30% contribution to the full $520 annual gross income instead of monthly SNAP net income after deductions, and it invented a $149 monthly minimum-benefit floor. The earned-income and standard deductions reduce net income to zero, and the resulting maximum monthly allotments total $3,596.04."
+us,scenario_066,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model applied an obsolete 15% Virginia refundable-EITC rate instead of the 2026 rate of 20%. Multiplying $39.78 by 20% and rounding yields $7.96, not $6.00."
+us,scenario_066,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly treated Virginia’s EITC as exclusively nonrefundable and denied the available refundable EITC. The household can elect the refundable credit, equal to 20% of its $39.78 federal EITC, producing $7.96."
+us,scenario_066,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model invented a minimum earned-income threshold and treated $520 of earnings as producing no childless federal EITC. The federal EITC phases in from the first dollar of earned income and equals $39.78 here, supporting a $7.96 Virginia refundable EITC."
+us,scenario_066,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model improperly rounded a positive federal EITC and its state percentage to zero and also used a 15% Virginia rate. The $39.78 federal EITC receives the 2026 refundable rate of 20%, yielding $7.96 regardless of the standard deduction or zero tax liability."
+us,scenario_066,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated Virginia’s EITC option as nonrefundable and limited it to tax liability. The household qualifies for the refundable option, which pays 20% of its $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model imposed a nonexistent roughly $600 earnings floor for the childless federal EITC. At $520 of earnings the federal credit is $39.78, and Virginia refunds 20% of that amount, rounded to $7.96."
+us,scenario_066,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model ignored the instruction to assume filing and program take-up and therefore refused to apply the elective refundable Virginia EITC. With the election assumed, the household receives 20% of its $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly classified Virginia’s EITC as solely nonrefundable. Virginia’s refundable EITC option applies to this eligible resident and pays 20% of the $39.78 federal EITC, yielding $7.96."
+us,scenario_066,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly identified the federal EITC base but applied a 15% Virginia rate. The 2026 refundable rate is 20%, so $39.78 × 0.20 rounds to $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The answer omitted the household’s qualifying Virginia refundable EITC. Virginia residency and a $39.78 federal EITC generate a refundable state credit of 20%, rounded to $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model applied a 15% Virginia refundable-EITC rate instead of the 2026 rate of 20% and rounded the federal base prematurely. Applying 20% to $39.78 yields $7.96.
+us,scenario_066,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a 15% Virginia refundable-EITC percentage. The applicable 2026 percentage is 20%, making the credit $39.78 × 0.20 = $7.956, rounded to $7.96."
+us,scenario_066,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used the wrong Virginia refundable-EITC rate of 15%. At the applicable 20% rate, the $39.78 federal EITC produces $7.96 after rounding."
+us,scenario_066,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model applied an obsolete 15% rate to the correct federal EITC base. Virginia’s 2026 refundable EITC is 20% of $39.78, which rounds to $7.96."
+us,scenario_066,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly made the absence of state tax liability disqualifying. Virginia’s refundable EITC can exceed liability, and this household receives 20% of its $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to trigger Virginia’s refundable EITC despite the household’s positive federal EITC and Virginia residency. The applicable computation is 20% of $39.78, rounded to $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model excluded Virginia’s refundable EITC based on the low income level, even though those earnings generate a $39.78 federal EITC. Virginia refunds 20% of that credit, producing $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model applied a 15% state percentage instead of Virginia’s 20% refundable-EITC rate for 2026. Using the unrounded $39.78 federal EITC base gives $7.96.
+us,scenario_066,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly concluded that a childless single adult with $520 of wages receives no Virginia refundable credit. The wages generate $39.78 of federal EITC, and the corresponding 20% Virginia refundable EITC is $7.96."
+us,scenario_066,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the absence of qualifying children as barring the Virginia refundable EITC. This 40-year-old qualifies for the childless federal EITC of $39.78, which generates a $7.96 Virginia refundable credit at 20%."
+us,scenario_066,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model wrongly limited refundable credits to the amount of state tax liability. Virginia’s refundable EITC is payable with zero liability and equals 20% of the household’s $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model omitted the Virginia refundable EITC for an eligible childless worker. The household’s $39.78 federal EITC produces a state refundable credit of $7.96 at the 20% rate.
+us,scenario_066,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,The model used a 15% Virginia refundable-EITC rate rather than the applicable 20% rate. Applying 20% to the precise $39.78 federal EITC yields $7.96.
+us,scenario_066,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Virginia has no applicable refundable state EITC. The household qualifies for the refundable option, calculated as 20% of its $39.78 federal EITC, or $7.96."
+us,scenario_066,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model applied a 15% Virginia rate and rounded the federal EITC base to $40. The 2026 refundable rate is 20%, and applying it to $39.78 yields $7.96."
+us,scenario_066,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated zero Virginia tax liability as eliminating the household’s credit eligibility. The refundable EITC is not liability-limited and pays 20% of the $39.78 federal EITC, producing $7.96."
+us,scenario_066,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model incorrectly ended Virginia’s refundable EITC after tax year 2025. The refundable credit applies in 2026 at 20% of the household’s $39.78 federal EITC, yielding $7.96."
+us,scenario_066,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model omitted the applicable Virginia refundable EITC. The eligible resident household has a $39.78 federal EITC, and Virginia refunds 20% of it, rounded to $7.96."
+us,scenario_066,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model wrongly required positive state tax liability or qualifying children for Virginia’s refundable EITC. The childless head qualifies through the federal EITC rules, and the refundable state amount is 20% of $39.78, or $7.96."
+us,scenario_066,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated a single-person, childless household as ineligible for Virginia’s refundable EITC. At age 40 with $520 of earnings, the head receives $39.78 of federal EITC and therefore $7.96 of refundable Virginia EITC at 20%."
+us,scenario_067,dependent1_medicaid_eligible,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model substituted the parents’ wages and pension for dependent1’s PolicyEngine MAGI income level. PolicyEngine places dependent1 in Indiana’s ACA adult category at 0.00 times FPL, so its separate discussion of SSI-related parental deeming and resources is irrelevant to the eligibility pathway used."
+us,scenario_067,dependent1_medicaid_eligible,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model treated the entire tax unit’s income as dependent1’s Medicaid MAGI. The applicable adult-expansion computation assigns dependent1 a MAGI income level of 0.00 times FPL, below 138%."
+us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,The model compared aggregate household income with Indiana’s Medicaid threshold instead of using dependent1’s computed MAGI income level. That level is 0.00 times FPL under the ACA adult expansion category.
+us,scenario_067,dependent1_medicaid_eligible,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model incorrectly attributed the parents’ combined income to dependent1 merely because dependent1 is in their tax household. PolicyEngine’s relevant MAGI calculation is 0.00 times FPL, which satisfies Indiana’s adult-expansion limit."
+us,scenario_067,dependent1_medicaid_eligible,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model used the tax unit’s $87,302 of wages and pension as dependent1’s Medicaid income. The adult-expansion test instead uses dependent1’s PolicyEngine MAGI income level of 0.00 times FPL."
+us,scenario_067,dependent1_medicaid_eligible,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the parents’ wages and pension and dependent1’s disability benefits into one $98,102 household-income figure, then tested that total against 138% FPL. PolicyEngine’s ACA adult calculation assigns dependent1 a MAGI income level of 0.00 times FPL, so that aggregation is the wrong income computation."
+us,scenario_067,dependent1_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model declined to apply Medicaid eligibility because SSI or other qualification details were not listed. SSI is unnecessary because dependent1 qualifies through Indiana’s ACA adult expansion category as an under-65 adult with a MAGI income level of 0.00 times FPL.
+us,scenario_067,dependent1_medicaid_eligible,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model assigned the parents’ income to dependent1 and then searched unnecessarily for an SSI or other categorical pathway. Indiana’s ACA adult expansion pathway applies directly, and dependent1’s relevant MAGI income level is 0.00 times FPL."
+us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used overall household income as the applicable Medicaid income measure. Dependent1’s PolicyEngine MAGI income level for the Indiana adult-expansion test is 0.00 times FPL, below the limit."
+us,scenario_067,dependent1_medicaid_eligible,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model applied household income and resources to unspecified income-tested pathways instead of the ACA adult expansion test actually controlling the result. Under that pathway, dependent1 has a MAGI income level of 0.00 times FPL, and the listed household resources are not the disqualifying test."
+us,scenario_067,dependent1_medicaid_eligible,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model treated disability benefits and household resources as disqualifying inputs without applying dependent1’s ACA adult MAGI calculation. PolicyEngine computes that relevant income level as 0.00 times FPL, and the adult expansion pathway does not use the cited household-resource shortcut."
+us,scenario_067,dependent1_medicaid_eligible,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model assigned roughly $87,000 of parental wages and pension to dependent1’s MAGI solely because dependent1 shares the tax unit. PolicyEngine’s adult-expansion computation gives dependent1 a MAGI income level of 0.00 times FPL, below 138%."
+us,scenario_067,dependent1_medicaid_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model falsely characterized Indiana as a non-expansion state and then applied disabled-adult income limits using parental household income. Indiana has adopted ACA Medicaid expansion, and dependent1 qualifies in its adult category with a MAGI income level of 0.00 times FPL."
+us,scenario_067,dependent1_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the $10,800 disability-benefit amount as income exceeding Indiana’s adult Medicaid limit. PolicyEngine’s MAGI computation for dependent1 yields 0.00 times FPL, so that benefit was incorrectly included in the controlling MAGI measure."
 us,scenario_067,dependent1_medicare_eligible,claude-opus-5,llm_error,age_disability,False,The model treated disabled status plus generic disability benefits as automatically establishing disability-based Medicare eligibility. Those facts do not establish SSDI entitlement and completion of the Medicare waiting period.
 us,scenario_067,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,The model improperly inferred that the unspecified disability benefits were SSDI and then treated the instruction that facts remain constant during the year as proof that the statutory waiting period had elapsed. Neither SSDI entitlement nor its required duration was supplied.
 us,scenario_067,dependent1_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,The model relabeled generic disability benefits as SSDI and assumed the Medicare waiting period was satisfied. Disabled status and an annual benefit amount alone do not establish the under-65 Medicare pathway.
@@ -4026,33 +4230,35 @@ us,scenario_067,dependent1_medicare_eligible,gpt-5.5,llm_error,age_disability,Fa
 us,scenario_067,dependent1_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,"The model assumed PolicyEngine converts disabled status plus any disability-benefit income directly into Medicare eligibility. The dependent has no specified SSDI entitlement, qualifying entitlement duration, or end-stage renal disease."
 us,scenario_067,dependent1_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model incorrectly asserted that PolicyEngine treats generic disability benefits as a Medicare-qualifying disability pathway. At age 23, the dependent lacks the specified SSDI entitlement and waiting-period history needed for that pathway."
 us,scenario_067,dependent1_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,The model collapsed disabled status and generic disability benefits into automatic Medicare eligibility. It failed to require SSDI entitlement followed by the statutory waiting period or another independently qualifying condition.
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The submitted $3,391 contradicts the model’s own recomputation of $5,347 and therefore does not follow from its stated tax calculation. It also incorrectly deducted $1,845 of auto-loan interest and used a $32,600 standard deduction instead of $32,200."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an obsolete estimated $28,500 standard deduction and invented $1,266 of unspecified nonrefundable credits. The calculation requires the $32,200 joint standard deduction and only the $500 Credit for Other Dependents."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model’s stated derivation yields approximately $5,620 after the $500 dependent credit, but it submitted $6,280 without a supporting computation. Using the exact $32,200 deduction and 2026 brackets yields $5,616.24."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from the mortgage balance and used that estimate in an itemized-deduction analysis even though unlisted interest is zero. Its submitted $5,594 also contradicts its own stated itemized result of about $6,722; the applicable deduction is the $32,200 standard deduction."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified $55,102 of taxable income, $6,116 of regular tax, and the $500 dependent credit, which leads directly to $5,616.24. It nevertheless submitted $6,104, a value unsupported by its own arithmetic."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly estimated the 2026 standard deduction and bracket threshold instead of applying the enacted $32,200 joint deduction and exact 2026 brackets. Its $30,000 deduction overstated taxable income by $2,200, causing the excessive liability."
-us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s stated tax calculation produced about $5,568 after the $500 credit, but it then submitted $10,500 based on nonexistent self-employment adjustments and benefit phase-outs. Neither item enters federal income tax before refundable credits for these facts."
-us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed TCJA expiration rules, substituting a $16,500 standard deduction, personal exemptions, and a 15% bracket. The applicable 2026 rules use a $32,200 standard deduction, 10% and 12% brackets at this income, and a $500 Other Dependent Credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly included 85% of the dependent’s disability benefits in the parents’ AGI and applied expired-law personal exemptions and rates. The dependent’s benefit is not part of the joint filers’ $87,302 gross income, and the return uses the $32,200 standard deduction plus a $500 Other Dependent Credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model inferred roughly $60,400 of taxable income using the wrong standard deduction and omitted the disabled adult dependent’s $500 nonrefundable credit. The correct taxable income is $55,102, followed by the $500 credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied assumed post-TCJA-expiration personal exemptions and a 15% bracket instead of the operative 2026 standard deduction and rate schedule. It also replaced the larger $32,200 standard deduction with unsupported itemized deductions."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model invented $4,500 of mortgage interest from a balance and applied personal exemptions under an inapplicable pre-TCJA regime. The household instead takes the $32,200 standard deduction, with no personal exemptions, and receives the $500 Other Dependent Credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $30,800 standard deduction and 2025 bracket threshold rather than the exact 2026 parameters. The $32,200 deduction reduces taxable income to $55,102 before the same $500 dependent credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer implies that the model used incorrect 2026 deduction or bracket parameters despite identifying the correct income base and $500 credit. Exact parameters produce $6,116.24 before the credit and $5,616.24 afterward."
-us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $30,000 estimated standard deduction and a 2025 bracket proxy. The exact 2026 joint deduction is $32,200, yielding $55,102 of taxable income and $5,616.24 after the $500 credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The unexplained $4,567 does not follow from the household’s $87,302 gross income, $32,200 standard deduction, and 2026 joint brackets. Those steps produce $6,116.24 before applying the $500 Other Dependent Credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized deductions based on mortgage and medical context rather than using only listed amounts and comparing them with the $32,200 standard deduction. The mortgage balance supplies no deductible interest, so taxable income is $55,102."
-us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $9,180 of the dependent’s disability benefits to the joint filers’ taxable income. Excluding that separate dependent income leaves $87,302 of gross income and $55,102 after the standard deduction."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The submitted $8,500 is unsupported by the stated generic calculation. Applying the $32,200 standard deduction to $87,302, the exact joint brackets, and the $500 Other Dependent Credit yields $5,616.24."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied personal exemptions and an expired-law deduction framework while also omitting the $500 Other Dependent Credit. The operative calculation uses the $32,200 standard deduction, no personal exemptions, and the $500 credit."
-us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed reversion to a $17,291 standard deduction and a 15% bracket, dramatically overstating taxable income and tax. It also omitted the $500 Other Dependent Credit available for the disabled adult dependent."
-us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model used an estimated $31,500 deduction and explicitly claimed no dependent, omitting the disabled 23-year-old as a qualifying dependent. That dependent produces a $500 nonrefundable Other Dependent Credit after regular tax is computed."
-us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly applied the $500 credit but used an estimated $30,800 standard deduction and estimated bracket threshold. Replacing them with the exact $32,200 deduction and 2026 schedule reduces the result to $5,616.24."
-us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model wrongly included $9,180 of the dependent’s disability benefits in the parents’ AGI and additionally deducted auto-loan interest. The joint filers’ gross income is $87,302, and no auto-loan-interest deduction enters this computation."
-us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model included the dependent’s $10,800 disability benefit in the parents’ AGI and awarded a $2,000 Child Tax Credit. At age 23 the dependent receives only the $500 Credit for Other Dependents, while the benefit is excluded from the joint filers’ gross income."
-us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the dependent’s $10,800 disability benefit as income of the joint filers and used an estimated $31,200 deduction. The benefit is not included in the parents’ $87,302 gross income, and the exact standard deduction is $32,200."
-us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented a $10,000 Child Tax Credit and used it to erase the liability. A disabled dependent age 23 does not meet the CTC age requirement and generates only the $500 nonrefundable Credit for Other Dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's submitted $3,391 contradicts its own final computation of $5,347. It also improperly deducted $1,845 of auto-loan interest and used a $32,600 standard deduction instead of $32,200; the trace uses $55,102 of taxable income and then subtracts the $500 dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $28,500 standard deduction instead of the 2026 joint deduction of $32,200, then invented approximately $1,266 of unspecified nonrefundable credits. The applicable nonrefundable credit is exactly the $500 Credit for Other Dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model's stated derivation reaches approximately $5,620 after the $500 dependent credit, but it submitted $6,280 without a supporting computation. The exact bracket calculation on $55,102 of taxable income yields $6,116.24 before the credit and $5,616.24 afterward."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from the mortgage balance even though unlisted interest must be zero, producing an invalid itemized-deduction analysis. It then submitted $5,594 despite its own calculation yielding about $6,722; the correct deduction is the $32,200 standard deduction."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly identified $55,102 of taxable income, approximately $6,116 of regular tax, and the $500 dependent credit, which directly yields approximately $5,616. It nevertheless submitted $6,104, failing to carry its own subtraction into the output."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a projected $30,000 standard deduction instead of the enacted 2026 joint deduction of $32,200. That inflated taxable income from $55,102 to $57,302; applying the exact brackets and then the correctly identified $500 dependent credit yields $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own tax computation produced about $6,068 before the $500 dependent credit and about $5,568 afterward, but it submitted $10,500 after invoking nonexistent self-employment adjustments and benefit phase-outs. Neither affects this federal income-tax output."
+us,scenario_067,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, restored personal exemptions, reduced the joint standard deduction to $16,500, and used a 15% bracket. The applicable 2026 rules use a $32,200 standard deduction, 10% and 12% brackets at this income, and a $500 Credit for Other Dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly included $9,180 of the dependent's disability benefits in the parents' AGI and applied obsolete personal exemptions and post-sunset rates. The tax unit's gross income is $87,302, and the current-law $32,200 standard deduction and $500 dependent credit control."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated the return as having no dependent credit even though the disabled 23-year-old qualifies for the $500 Credit for Other Dependents. It also used approximately $60,400 of taxable income rather than the $55,102 produced by the $32,200 joint standard deduction."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA sunset, personal exemptions, itemized deductions, and the pre-TCJA 15% bracket. The applicable computation uses the $32,200 joint standard deduction, current 10% and 12% brackets, and the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model fabricated $4,500 of mortgage interest from a mortgage balance, inserted an unlisted state-tax deduction, and restored personal exemptions under an inapplicable sunset scenario. The household instead takes the $32,200 standard deduction and receives the $500 Credit for Other Dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $30,800 standard deduction and 2025 bracket threshold rather than the 2026 $32,200 deduction and $24,800 top of the 10% bracket. Its dependent-credit treatment was correct, but its inflated taxable income produced the excess tax."
+us,scenario_067,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer implies an incorrect standard-deduction or bracket parameter despite correctly identifying AGI and the $500 dependent credit. With the $32,200 deduction, taxable income is $55,102 and exact regular tax is $6,116.24 before subtracting the credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted a $30,000 estimated standard deduction and a 2025 proxy bracket for the applicable 2026 parameters. The correct $32,200 deduction and $24,800 10% threshold yield $6,116.24 before the correctly identified $500 credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The submitted $4,567 has no computation supporting it and is $1,049.24 below the result of the applicable derivation. Gross income of $87,302 less the $32,200 standard deduction produces $55,102 taxable income, followed by $6,116.24 of tax and a $500 dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked itemized deductions from the mortgage and medical context without calculating any allowable itemized total, even though unlisted mortgage interest is zero and the standard deduction controls. The correct $32,200 standard deduction and $500 dependent credit yield $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $9,180 of the dependent's disability benefits to joint taxable income. Those benefits are not part of the filers' gross income, so taxable income is $55,102 rather than $64,282 before applying the $500 other-dependent credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The submitted $8,500 is unsupported by the model's generic statement and exceeds the regular tax before credits. The specified income, $32,200 joint standard deduction, exact 2026 brackets, and $500 dependent credit produce $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied an inapplicable TCJA sunset structure, invented a state-tax itemized deduction, restored personal exemptions, and omitted the $500 Credit for Other Dependents. The applicable current-law calculation instead uses the $32,200 standard deduction and 10%/12% brackets."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly used a $16,600 standard deduction, restored personal exemptions, applied a 15% bracket, and denied the dependent credit. The correct 2026 joint deduction is $32,200, and the disabled adult dependent generates a $500 nonrefundable Credit for Other Dependents."
+us,scenario_067,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed the TCJA provisions expired, reducing the standard deduction to $17,291 and applying a 15% bracket. It also omitted the $500 Credit for Other Dependents; the applicable rules use a $32,200 deduction and 10%/12% brackets."
+us,scenario_067,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used an approximate $31,500 standard deduction instead of $32,200 and failed to claim the disabled adult as a dependent for the $500 Credit for Other Dependents. Both errors raised its submitted liability above $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly applied the $500 dependent credit but used an estimated $30,800 standard deduction and an incorrect bracket threshold. The exact 2026 parameters are a $32,200 deduction and a $24,800 10% threshold, producing $5,616.24 after the credit."
+us,scenario_067,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly included $9,180 of the dependent's disability benefits in the joint AGI and deducted $1,845 of auto-loan interest. The filers' gross income is $87,302, no auto-interest deduction enters this trace, and the $32,200 standard deduction leaves $55,102 taxable."
+us,scenario_067,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly included the dependent's $10,800 disability benefits in the parents' AGI and awarded a $2,000 CTC to a 23-year-old. The dependent instead qualifies for the $500 Credit for Other Dependents, while the filers' gross income remains $87,302."
+us,scenario_067,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model reproduced the gross income, standard deduction, taxable income, brackets, and $6,116.24 regular tax exactly but failed to subtract the $500 Credit for Other Dependents. A disabled 23-year-old dependent qualifies for that nonrefundable credit even though the under-17 CTC is unavailable."
+us,scenario_067,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the dependent's $10,800 disability benefits in the parents' gross income and used a projected $31,200 standard deduction. The filers' gross income is $87,302, the applicable deduction is $32,200, and the correctly identified $500 dependent credit reduces tax to $5,616.24."
+us,scenario_067,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented a $10,000 CTC and used it to erase the liability. A 23-year-old dependent does not qualify for the under-17 CTC; the available nonrefundable credit is the $500 Credit for Other Dependents, reducing $6,116.24 to $5,616.24."
 us,scenario_067,federal_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the 23-year-old disabled dependent as a qualifying child for the refundable Additional Child Tax Credit. Disability can remove the age limit for the qualifying-child relationship test used for dependency and EITC purposes, but it does not override the CTC requirement that the child be under age 17, so the ACTC is $0."
 us,scenario_067,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented five qualifying children even though the household contains only one dependent, age 23, and then constructed a $10,000 CTC and $1,210 ACTC from that nonexistent child count. The sole dependent fails the CTC under-age-17 requirement, so there is no CTC remainder to refund and no ACTC."
 us,scenario_067,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 60 as above the Medicare eligibility threshold. The applicable age threshold is 65, and no disability-based Medicare pathway or other exception applies to the head, so the correct value is 0."
@@ -4069,64 +4275,67 @@ us,scenario_067,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The
 us,scenario_067,ssi,inkling,llm_error,categorical_eligibility,False,"The model assumed the disabled label made the dependent categorically eligible for SSI and calculated a residual benefit after the $20 exclusion. The dependent fails PolicyEngine’s SSI aged/blind/disabled test, so the correct computation ends at $0 before counting the $900 monthly disability benefit."
 us,scenario_067,ssi,kimi-k3,llm_error,categorical_eligibility,False,"The model treated general disability status as sufficient for SSI eligibility and focused on the rule ending parental deeming at age 18. The dependent is not SSI aged, blind, or disabled under PolicyEngine’s program-specific test, so neither non-deeming nor the income exclusion produces a benefit."
 us,scenario_067,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly declared the dependent SSI-qualified from the general disability fact and then offset disability income against a $1,913 monthly rate. PolicyEngine’s SSI categorical test is false for the dependent; additionally, $1,913 is not the applicable individual federal SSI maximum, so both the eligibility gate and downstream benefit-rate calculation were misapplied."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented a $500 disabled-dependent exemption and vague additional deductions, reducing taxable income below the correct $84,302. It also failed to apply the 2.95% rate consistently to the resulting tax base."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted a federal standard deduction for Indiana's $3,000 of base exemptions and invented health-premium, medical-expense, and dependent credits. Its final $4,168 also contradicts its own intermediate tax calculation of $1,356."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model overstated the dependent exemption and then selected an incorrect projected Indiana rate instead of 2.95%. The required calculation is $84,302 × 2.95% = $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the correct $84,302 tax base and $2,486.91 liability, then abandoned that result and submitted $2,620. Its submitted value follows neither of the alternative calculations stated in its explanation."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an approximate 2.9% rate rather than Indiana's 2.95% rate and reduced the tax base only approximately. Exact application of the $3,000 exemptions and 2.95% rate produces $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the entire $4,942 private pension and added a nonexistent $1,500 disabled-dependent exemption. It also used 3.05% instead of the 2026 rate of 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model invented $2,000 of additional dependent exemptions, reducing taxable income to $82,302 instead of $84,302. It then used 3.05% and submitted an estimate inconsistent with its own $2,510 calculation."
-us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $84,302 but applied a 3.0% rate. Indiana's 2026 rate is 2.95%, yielding $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented an additional $1,500 exemption for the disabled adult dependent, reducing taxable income to $82,802 instead of $84,302. It also applied 3.0% rather than 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model applied 3.05% and did not identify the required $3,000 exemption subtraction. The correct computation taxes $84,302 at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model overstated exemptions as approximately $4,500 rather than $3,000, producing an $82,802 tax base. Applying 2.95% to the correct $84,302 base yields $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income of $84,302 but used a 3.05% rate. The applicable 2.95% rate produces $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income of $84,302 but applied 3.0% instead of 2.95%. That rate error overstated the tax by $42.15."
-us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted $2,529.06 equals 3.0% of the correct $84,302 taxable income. Indiana's 2026 rate is 2.95%, so the liability is $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied an incorrect 2.75% rate directly to AGI and omitted the $3,000 base exemptions. Indiana taxes $84,302 after those exemptions at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model's unexplained $3,501 does not follow Indiana's flat-tax computation. The listed income produces $87,302 of AGI, $84,302 after base exemptions, and $2,486.91 of tax at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model improperly treated mortgage interest, medical expenses, a standard deduction, and unspecified credits as reductions in Indiana tax. The traced calculation uses only $3,000 of base exemptions before applying 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed $4,500 of personal and dependent exemptions instead of the applicable $3,000, understating taxable income by $1,500. The correct $84,302 base taxed at 2.95% yields $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model stated the correct $84,302 base but used 3.0% rather than 2.95% and also miscomputed 3.0% of that base. The correct multiplication is $84,302 × 2.95% = $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted $4,500 of exemptions by inventing an additional qualifying-child exemption for the 23-year-old dependent. Only $3,000 of base exemptions applies, leaving $84,302 taxable."
-us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model invented a $4,500 exemption total and referenced excluded Social Security or disability income that was never part of the filers' $87,302 AGI. The correct reduction is $3,000, leaving $84,302 taxable at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The unexplained $2,200 does not apply Indiana's 2.95% rate to the traced $84,302 taxable income. That computation yields $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $84,302 but applied 3.05% instead of 2.95%. This rate error produced approximately $2,571 rather than $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied 2.9% directly to the full $87,302 AGI and omitted the $3,000 base exemptions. Indiana instead taxes $84,302 at 2.95%."
-us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model allowed exemptions only for the two filers, omitting the dependent's $1,000 base exemption, and used 2.9% instead of 2.95%. Total base exemptions are $3,000, leaving $84,302 taxable."
-us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response could not be parsed as a substantive estimate."
-us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model overstated dependent exemptions by $2,000, subtracting both a $1,500 dependent exemption and a $1,500 disabled-child exemption. The applicable exemptions total $3,000, not $5,000."
-us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,900 of the dependent's disability benefits in the joint filers' AGI and invented a $3,800 Indiana standard deduction. The filers' AGI is $87,302, reduced only by $3,000 of base exemptions."
-us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model invented a $100 dependent credit and $374.42 of unspecified nonrefundable credits. No such reductions enter this calculation; $84,302 taxed at 2.95% remains $2,486.91."
-us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included the dependent's $10,800 disability benefits in the joint tax unit's AGI. Those benefits are not part of the filers' $87,302 AGI, which becomes $84,302 after the $3,000 exemptions."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model invented a $500 disabled-dependent exemption, reducing taxable income to $83,802 instead of $84,302, and then failed to apply the exact 2.95% rate. The correct computation is $84,302 × 2.95% = $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly started from federal taxable income after a federal standard deduction and invented Indiana health-premium, medical-expense, and dependent credits. Indiana instead taxes $87,302 less $3,000 of base exemptions at 2.95%, with none of those claimed adjustments."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model subtracted an inapplicable additional $1,500 dependent exemption and used projected rates of 3.05% or 3.12% instead of Indiana's 2026 rate of 2.95%. Taxable income is $84,302, not $83,802."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the exact computation—$87,302 less $3,000, multiplied by 2.95%, equals $2,486.91—but submitted $2,620 instead. Its final output discarded its own correct derivation."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an approximate 2.9% rate and an unspecified taxable-income approximation instead of calculating $84,302 at the exact 2.95% rate. That exact calculation yields $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented both a full $4,942 pension deduction for a 60-year-old taxpayer and a $1,500 disabled-dependent exemption. The pension remains in the $87,302 AGI, only $3,000 of base exemptions applies, and the resulting $84,302 is taxed at 2.95%, not 3.05%."
+us,scenario_067,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model inflated the dependent exemptions to $3,000 and reduced taxable income to $82,302 rather than subtracting only the $3,000 total base exemptions. It also used 3.05% instead of 2.95% and then submitted an amount inconsistent with its own estimated tax."
+us,scenario_067,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $84,302 but applied a 3.0% rate. Indiana's 2026 rate is 2.95%, producing $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented an additional $1,500 exemption for the disabled adult dependent, reducing taxable income to $82,802 instead of $84,302. It also applied 3.0% rather than the exact 2.95% rate."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer omitted the explicit $3,000 exemption step and described applying 3.05% directly to adjusted gross income. The required computation uses $84,302 after exemptions and the 2.95% rate, yielding $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted approximately $4,500 of exemptions instead of the $3,000 base exemptions. Although it used the correct 2.95% rate, applying it to the understated $82,802 taxable income produced $2,443 rather than $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income of $84,302 but used a 3.05% rate. The applicable 2026 Indiana rate is 2.95%, yielding $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly calculated taxable income of $84,302 but applied 3.0% instead of 2.95%. The correct tax is $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"Its $2,529.06 answer equals $84,302 taxed at 3.0%, so it used the wrong flat rate. Indiana's 2026 rate is 2.95%, producing $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a 2.75% rate and failed to subtract the $3,000 base exemptions before applying the tax rate. Indiana taxes $84,302 at 2.95%, not the full $87,302 at 2.75%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $3,501 does not follow from Indiana's $87,302 AGI, $3,000 base exemptions, and 2.95% rate. Those inputs produce taxable income of $84,302 and tax of $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model improperly invoked standard, mortgage-interest, medical-expense, and unspecified credit adjustments in estimating Indiana tax. The trace permits only $3,000 of base exemptions from $87,302 before applying 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model subtracted $4,500 of personal and dependent exemptions instead of the applicable $3,000 base exemptions. Its 2.95% rate was correct, but it applied that rate to $82,802 rather than $84,302."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,other,False,"The model stated the correct $3,000 exemption amount but made an arithmetic and output error: $84,302 × 3.0% is $2,529.06, not $2,619.06. It also used 3.0% instead of the applicable 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invented an additional $1,500 qualifying-child exemption and therefore used $82,802 of taxable income. Only $3,000 of base exemptions applies, leaving $84,302 to be taxed at 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used $4,500 of exemptions rather than the applicable $3,000 and introduced an irrelevant exclusion of purported taxable Social Security or disability benefits. The stated $87,302 AGI already consists only of wages and private pension, so taxable income is $84,302."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The unexplained $2,200 estimate does not implement the required $87,302 minus $3,000 calculation followed by the 2.95% rate. That computation yields $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly derived taxable income of $84,302 but applied a 3.05% rate. Indiana's applicable 2026 rate is 2.95%, yielding $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly subtracted $3,000 of exemptions but applied a 3.0% rate rather than 2.95%. The correct tax on $84,302 is $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied its rate to the full $87,302 AGI and omitted the $3,000 base exemptions. It also used 2.9% instead of 2.95%; the correct base is $84,302."
+us,scenario_067,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model allowed exemptions only for the two filers and omitted the dependent's $1,000 base exemption, leaving $85,302 instead of $84,302. It also used 2.9% rather than the exact 2.95% rate."
+us,scenario_067,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract.
+us,scenario_067,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model assigned $3,000 of exemptions to the dependent alone—$1,500 dependent plus $1,500 disabled-child exemptions—and subtracted $5,000 in total. Only $3,000 of total base exemptions applies, so taxable income is $84,302 rather than $82,302."
+us,scenario_067,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,900 of the dependent's disability benefits in joint AGI and invented a $3,800 Indiana standard deduction. Joint AGI is $87,302, and subtracting only $3,000 of base exemptions yields $84,302; it also used 3.05% instead of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model treated the dependent exemption as $1,500 and added another $1,500 disability exemption, producing $5,000 of total exemptions instead of $3,000. It also used a 2.9% rate instead of 2.95%."
+us,scenario_067,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly reached $84,302 but miscalled the $3,000 base exemptions a standard deduction, used 3.05% rather than 2.95%, and invented $474.42 of nonrefundable credits. No such credits reduce this output, so $84,302 × 2.95% remains $2,486.91."
+us,scenario_067,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model improperly added the adult dependent's $10,800 disability benefits to the couple's joint AGI. The dependent's income is not part of their return; AGI is $87,302, taxable income after $3,000 of exemptions is $84,302, and the applicable rate is 2.95%, not 3.05%."
 us,scenario_067,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It incorrectly added $67,750 of educational assistance to wages after imposing the IRC §127 exclusion cap, inflating AGI from $34,083.53 to $101,834. Its submitted $8,258 also contradicts its own computed tax of $13,573."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"It improperly used the refundable EITC to reduce an output explicitly measured before refundable credits and invented additional reductions for employer health premiums. It also annualized wages from hours despite the prompt stating that $34,084 was the full annual wage total."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It incorrectly treated $67,750 of the educational-assistance input as taxable wages under IRC §127. That amount does not enter this federal AGI calculation, which uses $34,083.53 of wages alone."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"It incorrectly added educational assistance above $5,250 to taxable wages, producing $101,834 of AGI instead of $34,083.53. The resulting $85,734 taxable-income base is therefore overstated."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"Its reasoning reaches the correct $17,984 taxable-income base and even computes approximately $1,910, but it submits $2,372. It therefore failed to carry its own 2026 single-bracket calculation into the final value."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It incorrectly included $67,750 of educational assistance in AGI under the IRC §127 cap. It also substituted estimated deductions and bracket thresholds instead of the applicable $16,100 standard deduction and 2026 rate schedule."
-us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"After abandoning its erroneous educational-assistance inclusion, it used the correct wage-only taxable-income concept but replaced the bracket calculation with an unsupported $1,750 estimate. Taxable income of $17,983.53 produces $1,910.02 under the applicable 2026 brackets."
-us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It incorrectly included $67,750 of educational assistance and separately deducted $8,389 of employer-sponsored insurance premiums from the stated annual wages. Neither adjustment belongs in this traced AGI, and it also used an estimated $15,400 deduction instead of $16,100."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It applied a post-TCJA-sunset regime with a personal exemption, a $7,600 standard deduction, and 15% brackets instead of the applicable 2026 law. It also incorrectly subtracted the separately listed insurance premium from wages."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"It invented a Child Tax Credit or other nonrefundable credit even though this single adult has no dependent or stated credit-qualifying expense. It also used a $15,200 standard deduction and taxed all taxable income at 10% rather than applying the $16,100 deduction and graduated rates."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It incorrectly subtracted $8,389 of employer-sponsored insurance premiums from gross wages and applied a sunset-law deduction-plus-personal-exemption regime. The traced AGI remains $34,083.53 and receives the $16,100 standard deduction."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It added all $73,000 of educational assistance to income, deducted the insurance premium, and applied an inapplicable post-sunset personal-exemption regime. The educational-assistance and insurance inputs do not alter the traced $34,083.53 federal AGI."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"It used an inapplicable post-TCJA-sunset standard deduction, personal exemption, and 15% bracket. The applicable $16,100 standard deduction leaves $17,983.53 taxable and the 2026 main rates yield $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $11,116 answer implies that it treated a substantial portion of the $73,000 educational-assistance input as taxable or otherwise inflated the tax base. The traced computation includes only $34,083.53 of wages in AGI and yields $17,983.53 of taxable income."
-us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It incorrectly added $67,750 of educational assistance to taxable wages. It compounded that error by using the 2024 standard deduction and brackets as a proxy rather than the applicable 2026 $16,100 deduction and rate thresholds."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It incorrectly concluded that the standard deduction eliminated tax on $34,084 of wages. The $16,100 deduction leaves $17,983.53 taxable, producing $1,910.02 before refundable credits."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It incorrectly reduced the liability to zero after the standard deduction. The deduction does not exhaust the $34,083.53 AGI; $17,983.53 remains taxable."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"It applied a projected post-sunset deduction, personal exemption, and 15% bracket instead of the applicable 2026 rules. The correct deduction is $16,100, with no personal exemption, and the resulting main-rate tax is $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It incorrectly included $67,750 of educational assistance above the IRC §127 cap as taxable income. That input does not enter the traced federal AGI, which consists of $34,083.53 in wages."
-us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It incorrectly included the entire $73,000 educational-assistance amount in income. Removing that amount leaves $34,083.53 of AGI and $17,983.53 after the standard deduction."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its statement that taxable income was about $19,000 reflects use of a smaller standard deduction than the applicable $16,100 amount. The correct taxable income is $17,983.53, and the graduated-rate tax is $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It correctly excluded educational assistance but used a 2025-like $15,000 standard deduction and 2025-like bracket thresholds. Applying the 2026 $16,100 deduction and applicable brackets reduces the result to $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It incorrectly deducted $8,389 of employer-sponsored insurance premiums from the stated annual wages and then applied a post-sunset deduction and 15% bracket. The traced AGI is the full $34,083.53 wage amount and receives a $16,100 standard deduction."
-us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the required structured answer was missing."
-us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"It correctly excluded educational assistance but used an estimated $15,750 standard deduction and estimated bracket thresholds. The applicable $16,100 deduction produces $17,983.53 of taxable income and $1,910.02 of tax."
-us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"It correctly kept educational assistance out of AGI but used a $15,750 standard deduction and incorrect bracket thresholds. The applicable $16,100 deduction and 2026 brackets yield $1,910.02."
-us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"It improperly subtracted the refundable EITC when calculating an output defined before refundable credits and also asserted that the standard deduction reduced taxable income to zero. The $16,100 deduction leaves $17,983.53 taxable, while EITC belongs in the separate refundable-credits output."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance above the IRC §127 exclusion to wages, while this input does not enter the federal AGI computation here. It also submitted $8,258 despite its own calculation yielding $13,573; the correct taxable income is $17,983.53 and its tax is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model improperly reduced the stated annual wages using weeks unemployed and then subtracted refundable EITC when the requested quantity is explicitly before refundable credits. It also reversed the §127 treatment and speculated about health-related reductions; AGI remains $34,083.53, taxable income is $17,983.53, and pre-refundable-credit tax is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $67,750 of the separately reported educational-assistance input as additional taxable wages. That amount does not enter AGI here, so the $16,100 standard deduction applies to $34,083.53 and leaves $17,983.53 taxable."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly added educational assistance above $5,250 to federal gross income. PolicyEngine's federal calculation uses $34,083.53 of employment income alone, yielding $17,983.53 after the $16,100 standard deduction and $1,910.02 of tax."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached the correct $17,984 taxable-income calculation and even derived approximately $1,910, but then submitted $2,372 without a supporting rate calculation. Applying the 2026 single main-rate schedule to $17,983.53 yields $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $67,750 of educational assistance in AGI and used an estimated $15,750 deduction instead of the applicable $16,100 standard deduction. Federal AGI is $34,083.53 and taxable income is $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"After eventually choosing the correct pathway of excluding the educational-assistance input, the model replaced its own bracket result near $1,915 with an unsupported $1,750 estimate. The exact $16,100 deduction and 2026 single brackets produce $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model both added $67,750 of educational assistance to wages and subtracted $8,389 of ESI premiums from already stated gross wages. Neither adjustment belongs in this AGI computation; $34,083.53 less the $16,100 standard deduction leaves $17,983.53 taxable."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset regime with a $7,600 standard deduction, personal exemption, and 15% bracket, instead of the applicable 2026 $16,100 standard deduction and main-rate schedule. It also improperly subtracted ESI premiums from the stated wages."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model invented a nonrefundable child or other credit even though this single-person household has no dependent or stated credit-qualifying facts. No nonrefundable credit reduces the $1,910.02 tax generated from $17,983.53 of taxable income."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $8,389 of employer-sponsored insurance premiums from stated wages and applied an inapplicable post-sunset deduction-plus-exemption regime. The calculation instead uses $34,083.53 of AGI and a $16,100 standard deduction."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly added all $73,000 of educational assistance to income, subtracted ESI premiums from wages, and applied a reverted deduction, exemption, and rate schedule. The educational-assistance input does not enter federal AGI here, which remains $34,083.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an inapplicable post-TCJA-sunset $8,300 standard deduction, personal exemption, and 15% bracket. The applicable $16,100 standard deduction leaves $17,983.53 taxable and the 2026 main rates yield $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The unsupported $11,116 answer implies that the model treated some of the educational-assistance input as taxable or otherwise inflated taxable income. The correct derivation uses $34,083.53 of AGI, subtracts $16,100, and taxes $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance to taxable wages and used a 2024 deduction and brackets as proxies. The assistance does not enter federal AGI here, and the applicable 2026 standard deduction is $16,100."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that $34,084 was fully sheltered by the standard deduction. The $16,100 deduction leaves $17,983.53 of positive taxable income, producing $1,910.02 before refundable credits."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the liability to zero after the standard deduction. Wages exceed the $16,100 deduction by $17,983.53, and the resulting main-rate tax is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model applied an inapplicable post-sunset standard-deduction-plus-personal-exemption regime and a 15% marginal bracket. The applicable 2026 deduction is $16,100, with no personal exemption in this computation, and the main rates yield $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $67,750 of educational assistance above an assumed §127 exclusion in taxable income. This input does not enter federal AGI here, so taxable income is $17,983.53 rather than approximately $85,734."
+us,scenario_068,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the entire $73,000 educational-assistance amount in income. Federal AGI consists of $34,083.53 of employment income, and the $16,100 standard deduction leaves $17,983.53 taxable."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used an unspecified deduction that left about $19,000 taxable rather than applying the exact $16,100 standard deduction. The resulting taxable income is $17,983.53, whose tax is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly excluded the educational-assistance input but used a 2025-like $15,000 standard deduction and 2025-like bracket thresholds. The applicable 2026 deduction is $16,100, leaving $17,983.53 taxable and $1,910.02 of tax."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied an inapplicable post-TCJA-sunset standard deduction, personal exemption, and 15% bracket. Under the applicable 2026 rules, the single standard deduction is $16,100 and tax on $17,983.53 is $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $8,389 of ESI premiums from stated gross wages and then used an inapplicable post-sunset deduction and rate schedule. AGI remains $34,083.53, and the $16,100 standard deduction produces $17,983.53 of taxable income."
+us,scenario_068,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_068,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model correctly excluded the educational-assistance input but used an estimated $15,750 standard deduction and approximate thresholds. The applicable $16,100 deduction and exact 2026 main-rate schedule produce $1,910.02."
+us,scenario_068,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly kept the educational-assistance input out of AGI but used a $15,750 standard deduction rather than $16,100 and approximate bracket thresholds. Those parameter errors raised taxable income to $18,334 instead of $17,983.53."
+us,scenario_068,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model improperly subtracted EITC in determining taxable income and the requested pre-refundable-credit liability. EITC is refundable and excluded from this output; the standard deduction leaves $17,983.53 taxable and $1,910.02 due before refundable credits."
 us,scenario_068,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly projected the temporary American Rescue Plan expansion of the childless EITC into 2026 and therefore used a phaseout endpoint near $33,000–$34,000. Under 2026 law, the temporary expansion is not in effect, and $34,084 exceeds the applicable childless EITC income limit, yielding zero EITC."
 us,scenario_068,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated wages of $34,084 and a one-person household as qualifying for EITC without applying the childless single-filer phaseout endpoint. At that income the childless EITC is fully phased out, and no other refundable credit applies."
 us,scenario_068,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, violating the required output contract."
@@ -4145,65 +4354,69 @@ us,scenario_068,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
 us,scenario_068,payroll_tax,inkling,llm_error,state_local_rule,False,"The model correctly included employee Social Security and Medicare but incorrectly added a 0.45% Maryland FAMLI contribution. No such mandatory Maryland employee payroll tax applies to this 2026 reference calculation, leaving only the two federal components totaling $2,607.39."
 us,scenario_068,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so it failed the required structured-output contract."
 us,scenario_068,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model identified the correct Social Security and Medicare components but rounded their sum to $2,600 instead of adding the liabilities at monetary precision. The traced components are $2,113.18 and $494.21, which total $2,607.39."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added $67,750 of educational assistance to Maryland AGI and consequently reduced the personal exemption. The computation uses AGI of $34,083.53, a $3,400 standard deduction, and the full $3,200 exemption."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted $8,389 of employer-sponsored insurance premiums from the stated wages and then asserted that ordinary Maryland deductions eliminated the remaining income. PolicyEngine uses AGI of $34,083.53 and taxable income of $27,483.53, which remains subject to Maryland tax."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $1,029 contradicts every calculation in the model's own reasoning, which produced either about $4,554 or $1,336. It also used a $2,700 standard deduction instead of $3,400; applying the traced AGI and deductions yields $27,483.53 of taxable income and $1,252.97 of tax."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an estimated $2,700 Maryland standard-deduction cap instead of the applicable $3,400 deduction. It also misstated the bracket base and then added an unsupported adjustment after computing tax."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used a roughly $2,700 standard deduction rather than $3,400, leaving taxable income about $700 too high. Its stated tax arithmetic also fails to apply the Maryland schedule consistently to the resulting base."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $67,750 of the separately listed educational assistance as additional taxable Maryland income. PolicyEngine's Maryland AGI is $34,083.53, and the applicable standard deduction is $3,400 rather than $2,500."
-us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly left educational assistance out of the operative AGI but used an estimated $2,550 standard deduction instead of $3,400. It also understated the tax generated by its own stated $28,334 taxable-income figure."
-us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The unexplained $93,445 AGI improperly incorporates amounts beyond the traced $34,083.53 Maryland AGI. The correct deductions are specifically $3,400 and $3,200, leaving $27,483.53 rather than $87,465 taxable."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced the $34,084 wages by the $8,389 employer-sponsored insurance premium to obtain a false $25,695 AGI. It also used a $2,550 standard deduction instead of $3,400."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an approximate shortcut from gross wages rather than the specified Maryland computation. Subtracting the $3,400 standard deduction and $3,200 exemption gives $27,483.53 taxable, to which the schedule applies for $1,252.97."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,The model improperly included an estimated county tax in state_income_tax_before_refundable_credits even though the requested output excludes local income tax. It also reduced AGI by insurance premiums and used the wrong Maryland standard deduction.
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $98,695 Maryland AGI, reflecting improper inclusion of educational assistance net of other amounts. The traced AGI is $34,083.53, and the deduction is $3,400 rather than $2,550."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $2,550 standard deduction instead of $3,400, overstating taxable income by $850. Maryland taxable income is $27,483.53, not $28,334."
-us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $4,786 answer is consistent with treating the large educational-assistance amount as state-taxable income. PolicyEngine uses only $34,083.53 of Maryland AGI before the $3,400 deduction and $3,200 exemption."
-us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used projected deduction and exemption figures instead of the applicable $3,400 and $3,200 amounts. More decisively, its $1,453.28 does not follow Maryland's stated brackets for its own $28,034 taxable-income estimate, which would produce about $1,279."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model falsely asserted that deductions and exemptions fully offset $34,084 of income. They total $6,600, leaving $27,483.53 of taxable income rather than zero."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly floored Maryland tax at zero despite $27,483.53 remaining after the $3,400 standard deduction and $3,200 exemption. No unspecified nonrefundable credits can be assumed to erase that liability."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $2,900 standard deduction instead of $3,400. That left taxable income $500 too high and produced a correspondingly overstated tax."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly increased AGI to $101,834 by treating educational assistance above $5,250 as additional taxable income. PolicyEngine uses Maryland AGI of $34,083.53 before the $3,400 deduction and $3,200 exemption."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used a projected $2,850 standard deduction rather than the applicable $3,400 amount. This overstated taxable income by $550 and therefore overstated the scheduled tax."
-us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model improperly included educational assistance in Maryland AGI and also stated taxable income that is arithmetically inconsistent with its claimed deductions. The traced taxable income is $27,483.53, not $101,334."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used a rough effective-tax estimate instead of applying the exact Maryland deductions and graduated schedule. Taxable income is $27,483.53, and the schedule produces $1,252.97 rather than approximately $1,400."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $2,350 Maryland standard deduction instead of $3,400. This overstated taxable income by $1,050 and raised the tax by $49.88."
-us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted employer-sponsored insurance premiums to reduce AGI to $25,695. It also used a $2,400 standard deduction instead of $3,400; the traced taxable income is $27,483.53."
-us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model substituted approximate deductions of $2,650 and $3,400 for the applicable $3,400 standard deduction and $3,200 personal exemption. The exact deductions leave $27,483.53 taxable and produce $1,252.97."
-us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the entire $73,000 educational-assistance input to wages because no education expenses were listed. PolicyEngine's Maryland AGI remains $34,083.53, and Maryland taxable income after the $3,400 deduction and $3,200 exemption is $27,483.53."
-us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an indexed $2,850 standard-deduction cap instead of the applicable $3,400 deduction. This left taxable income $550 too high."
-us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model subtracted only an estimated $2,000 standard deduction and omitted the $3,200 personal exemption from its taxable-income calculation. The two applicable subtractions total $6,600 and leave $27,483.53 taxable."
-us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model invented a personal-exemption credit and a standard-deduction credit after already subtracting those items from income, double counting their tax benefit. It also used a $3,100 standard deduction instead of $3,400; Maryland applies the schedule directly to $27,483.53 here with no such extra credits."
-us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated the EITC as a deduction that reduced Maryland taxable income to zero. Taxable income is determined first as $34,083.53 minus the $3,400 standard deduction and $3,200 exemption, and refundable credits do not erase the requested pre-refundable-credit tax."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model added $67,750 of educational assistance to AGI even though the engine's Maryland AGI contains only $34,083.53, then used an incorrect $2,700 standard deduction and reduced the personal exemption. The correct deductions are $3,400 and $3,200, leaving $27,483.53 taxable."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from the separately supplied gross wages and then asserted that the remaining income was eliminated by deductions. Maryland taxable income is $27,483.53 after the $3,400 standard deduction and $3,200 exemption, not zero."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The submitted $1,029 contradicts both computations in the model's own reasoning, which produced $4,554 under its taxable-assistance interpretation and $1,336 under its excluded-assistance interpretation. It never computed the traced pathway of $34,083.53 AGI less $3,400 and $3,200, followed by Maryland's rate schedule."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an outdated estimated standard-deduction cap of $2,700 instead of the applicable $3,400, overstating taxable income by $700. It also appended an unexplained adjustment from its calculated $1,286 to $1,308 even though the personal exemption is a deduction already included in taxable income."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an approximate $2,700 standard deduction rather than $3,400 and described the $3,200 personal exemption inconsistently as a minimal tax effect. The exemption and standard deduction reduce Maryland taxable income to $27,483.53 before the rate schedule is applied."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated $67,750 of educational assistance as income in addition to the listed wages, inflating Maryland AGI to $101,834. The traced AGI is $34,083.53, and the model also used an obsolete $2,500 standard-deduction cap instead of $3,400."
+us,scenario_068,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model ultimately used the wage-only AGI, it substituted an approximate $2,550 standard deduction for the applicable $3,400 and then miscomputed the tax on its own stated $28,334 taxable income. The correct taxable income is $27,483.53 and yields $1,252.97."
+us,scenario_068,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented Maryland AGI of $93,445 rather than using the traced $34,083.53 and estimated a combined deduction amount rather than applying $3,400 plus $3,200. This inflated taxable income from $27,483.53 to $87,465."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced the supplied $34,084 wages by the $8,389 employer-sponsored insurance premium to produce an incorrect $25,695 AGI. It also used a $2,550 standard deduction instead of $3,400; the traced taxable income is $27,483.53."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an approximate Maryland calculation from gross wages but does not apply the traced $3,400 standard deduction, $3,200 exemption, and rate schedule. Those steps produce $27,483.53 of taxable income and $1,252.97 of tax, not $1,133."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model both reduced AGI to $25,695 by subtracting employer insurance premiums and added an estimated county income tax to the requested state-only output. County tax is excluded from this variable, while Maryland state tax uses $34,083.53 AGI less $3,400 and $3,200."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported AGI of $98,695, effectively importing educational assistance into taxable income instead of following the traced $34,083.53 AGI. It also used a $2,550 standard deduction rather than $3,400 and misstated the base tax below $3,000."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used a $2,550 standard-deduction cap instead of the applicable $3,400, leaving taxable income $850 too high. Maryland taxable income is $27,483.53 after the standard deduction and $3,200 exemption."
+us,scenario_068,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $4,786 answer is consistent with taxing the educational assistance in addition to wages rather than using the traced $34,083.53 AGI. Applying the $3,400 standard deduction and $3,200 exemption to the traced AGI yields $27,483.53 taxable and $1,252.97 of state tax."
+us,scenario_068,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model substituted projected deductions of $2,750 and $3,300 for the applicable $3,400 standard deduction and $3,200 personal exemption, then misapplied the rate schedule: its stated $28,034 taxable income would not produce $1,453.28 under Maryland's brackets. The traced taxable income is $27,483.53."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions and exemptions fully offset $34,084 of wages. The $3,400 standard deduction and $3,200 exemption leave $27,483.53 taxable, producing $1,252.97."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model floored the liability at zero without subtracting the actual deductions or applying Maryland's brackets. Only $6,600 is deducted from $34,083.53 of AGI, leaving $27,483.53 taxable."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model estimated the standard deduction at $2,900 instead of using $3,400, overstating taxable income by $500. With the $3,200 exemption as well, taxable income is $27,483.53 and the schedule yields $1,252.97."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model added the educational-assistance amount above a presumed exclusion to wages and used $101,834 as AGI. The traced Maryland AGI is $34,083.53; after the $3,400 standard deduction and $3,200 exemption, only $27,483.53 is taxable."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used a projected $2,850 standard deduction rather than the applicable $3,400. This overstated taxable income by $550 and therefore overstated the tax produced by Maryland's schedule."
+us,scenario_068,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model imported educational assistance into Maryland income and reported taxable income of $101,334, which also does not reconcile with its stated $2,550 deduction and $3,200 exemption. The traced AGI is $34,083.53 and taxable income is $27,483.53 after the correct deductions."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an unsupported rough approximation instead of the applicable deduction amounts and Maryland rate schedule. The exact pathway is $34,083.53 less $3,400 and $3,200, producing $27,483.53 taxable and $1,252.97 of tax."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $2,350 maximum standard deduction rather than the 2026 amount of $3,400. That inflated taxable income by $1,050; the correct taxable income is $27,483.53."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model approximated the standard deduction as $2,600 instead of applying $3,400. With the $3,200 personal exemption, Maryland taxable income is $27,483.53 rather than about $28,284."
+us,scenario_068,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from the separately reported wages to derive $25,695 of AGI. It then used a $2,400 standard deduction instead of $3,400; the correct taxable income is $27,483.53."
+us,scenario_068,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used approximate deductions of $2,650 and $3,400 instead of the applicable $3,400 standard deduction and $3,200 exemption. Those correct amounts leave $27,483.53 taxable and produce $1,252.97."
+us,scenario_068,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model added the entire $73,000 educational-assistance input to wages because no education expenses were listed, whereas the traced Maryland AGI is $34,083.53. It also used a $2,700 standard deduction and an inapplicable estimated rate schedule, greatly overstating liability."
+us,scenario_068,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an indexed standard-deduction cap of $2,850 rather than the applicable $3,400. The additional $550 deduction reduces taxable income to $27,483.53 and tax to $1,252.97."
+us,scenario_068,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used only an approximate $2,000 standard deduction and failed to subtract Maryland's $3,200 personal exemption in its taxable-income calculation. Both deductions apply, leaving $27,483.53 taxable rather than $32,084."
+us,scenario_068,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $2,350 standard-deduction maximum instead of $3,400. This overstated taxable income by $1,050; the correct base after the $3,200 exemption is $27,483.53."
+us,scenario_068,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model first used the wrong $3,100 standard deduction, then invented separate tax credits for the personal exemption and standard deduction after already subtracting them from income. Maryland treats the $3,400 standard deduction and $3,200 exemption as reductions in deriving $27,483.53 taxable income, not additional nonrefundable credits."
+us,scenario_068,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated EITC and the standard deduction as eliminating Maryland taxable income and liability. EITC does not reduce taxable income in this calculation, and the actual $3,400 deduction plus $3,200 exemption leave $27,483.53 taxable."
 us,scenario_068,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model correctly recognized that the childless federal EITC phases out to zero at this income but then invented a $245 residual Maryland refundable credit. Applying Maryland’s percentage to a federal EITC of zero yields $0, not a minimal positive amount."
 us,scenario_068,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated $34,084 as qualifying low income without applying the federal childless EITC phaseout ceiling, then assigned a Maryland EITC amount exceeding the credit pathway it cited. With no qualifying children, federal EITC is zero at this income, so the percentage-based Maryland refundable EITC and the total state refundable credits are both $0."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own derivation reaches approximately $1,980 after the $192.40 Saver's Credit, but it submitted $1,697. This is an unsupported final-value substitution inconsistent with its stated calculation."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of the 2026 single amount of $16,100 and did not subtract the $192.40 Saver's Credit. The correct taxable income is $20,169.67, with $2,172.36 of tentative tax reduced to $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived approximately $2,172 of tentative tax but arbitrarily replaced it with $2,398 and omitted the $192.40 Saver's Credit. Subtracting that nonrefundable credit yields $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The stated bracket arithmetic equals about $2,172, not $2,549. The model also incorrectly asserted that no nonrefundable credit applies, overlooking the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated tax calculation produces about $2,076 under its assumed parameters, not the submitted $2,544, and it omitted the Saver's Credit. Using the $16,100 standard deduction and subtracting the $192.40 credit gives $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used speculative standard-deduction and bracket estimates instead of the 2026 parameters and then declared that no credits apply. At AGI $36,269.67, the filer qualifies for a 10% Saver's Credit on $1,924 of contributions, reducing $2,172.36 to $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly computed tentative tax near $2,172 but supplied $1,329 without any calculation supporting the additional $843 reduction. The applicable Saver's Credit is only $192.40, producing $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $600 of health-insurance premiums from AGI and used the wrong standard deduction and bracket threshold. It also counted only $1,554 of contributions for the Saver's Credit, excluding the qualified Roth contributions; the credit base is $1,924."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied an assumed TCJA sunset, replacing the 2026 $16,100 standard deduction and 10%/12% brackets with a personal exemption and a 15% bracket. The applicable parameters produce $2,172.36 before the correctly identified $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction, used a $15,500 standard deduction instead of $16,100, and did not account for the $192.40 Saver's Credit. Those corrections produce $20,169.67 of taxable income and $1,979.96 after nonrefundable credits."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model subtracted a personal exemption that does not enter this 2026 computation and did not show bracket arithmetic capable of producing $2,576. The correct calculation uses only the $16,100 standard deduction, then subtracts the $192.40 Saver's Credit from $2,172.36."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly deducted health-insurance premiums in deriving AGI and applied a reverted-law standard deduction, personal exemption, and tax schedule. AGI is $36,269.67 and taxable income is $20,169.67 under the applicable 2026 rules."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $15,700 standard deduction and the 2025 $11,925 bracket threshold instead of the 2026 $16,100 deduction and applicable brackets. Its Saver's Credit calculation was correct, but tentative tax is $2,172.36 rather than $2,229.90."
-us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model invoked a standard deduction plus personal exemption and produced $2,780.50 of implied tentative tax without valid bracket arithmetic. The 2026 computation uses a $16,100 standard deduction, produces $2,172.36 of tax, and subtracts the stated $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated parameters—a $15,420 standard deduction and $12,259 bracket threshold—instead of the 2026 $16,100 standard deduction and applicable bracket schedule. Its $192.40 Saver's Credit was correct, but it applied that credit to an overstated tentative tax."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented child, dependent-care, or other nonrefundable credits despite the household having no dependents or qualifying care facts. Only the $192.40 Saver's Credit offsets the $2,172.36 tentative tax, so liability is not zero."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed unspecified nonrefundable credits fully eliminated the tax, contrary to the instruction that unlisted facts are false. The only applicable nonrefundable credit is the $192.40 Saver's Credit, leaving $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied a projected post-TCJA-sunset personal exemption, reduced standard deduction, and 15% bracket. It also missed the $192.40 Saver's Credit; the applicable 2026 10%/12% schedule produces $1,979.96 after that credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly reached the tentative tax of about $2,172.40 but incorrectly stated that no nonrefundable credit was available. AGI $36,269.67 qualifies for a 10% Saver's Credit on $1,924, reducing tax by $192.40."
-us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model stopped at tentative regular tax and failed to apply the Saver's Credit. The qualified contribution base is $1,924 and the 10% credit is $192.40, leaving $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The submitted $1,623 is unsupported by the model's abbreviated derivation and does not follow from the applicable deductions, brackets, or credit. Taxable income of $20,169.67 produces $2,172.36 before the $192.40 Saver's Credit."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied assumed sunset rules with an $8,300 standard deduction, a personal exemption, and a 15% bracket. The applicable 2026 calculation uses the $16,100 standard deduction and 10%/12% brackets; its $192.40 Saver's Credit calculation was otherwise correct."
-us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted health-insurance premiums from AGI, used a $15,400 standard deduction, and excluded Roth contributions from the Saver's Credit base. AGI is $36,269.67, the standard deduction is $16,100, and the credit is $192.40 on $1,924."
-us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"The model treated this single person with no qualifying dependent as head of household, producing an inapplicable $21,900 deduction and head-of-household brackets. The correct filing status is single, and Roth as well as traditional contributions enter the $1,924 Saver's Credit base."
-us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model estimated positive tentative tax and a modest Saver's Credit, then asserted without any eligible credit or arithmetic that other nonrefundable credits reduce the result to zero. Only $192.40 is subtracted from $2,172.36, leaving $1,979.96."
-us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model omitted the $276 student-loan-interest deduction from AGI and applied a 50% Saver's Credit rate instead of 10% at AGI $36,269.67. It also excluded Roth contributions from the credit base and submitted $1,213.50 despite its own arithmetic yielding $1,475.52."
-us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a Credit for Other Dependents even though the household contains no dependent. The filer instead receives only the $192.40 Saver's Credit against $2,172.36 of tentative tax, so the result is not zero."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning correctly derived approximately $2,172 of tentative tax and a $192 Saver's Credit, which yields approximately $1,980. It then submitted $1,697, a value unsupported by its own computation."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of the 2026 $16,100 single-filer deduction and did not subtract the $192.40 Saver's Credit. Correct taxable income is $20,169.67, tentative tax is $2,172.36, and tax after that credit is $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly computed about $2,172.40 of tentative tax from approximately $20,170 of taxable income, but then submitted $2,398 without an arithmetic basis. It also failed to subtract the $192.40 Saver's Credit required to reach $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model added the two bracket components incorrectly: $1,240 plus about $932 equals about $2,172, not $2,549. It also incorrectly stated that no nonrefundable credit applies, omitting the $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated bracket calculation produced about $2,076 using its own erroneous deduction estimate, yet it submitted $2,544 without support. It also omitted the Saver's Credit and used the wrong 2026 standard deduction; the correct sequence yields $2,172.36 before the $192.40 credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model repeatedly substituted projected standard deductions for the enacted 2026 $16,100 amount and ended with $15,700, overstating taxable income by $400. It also declared that no credit applied, omitting the $192.40 Saver's Credit on $1,924 of qualified contributions."
+us,scenario_070,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived about $2,172 of tentative tax but then jumped to $1,329 after invoking the Saver's Credit without computing that result. At the applicable 10% rate, the credit is only $192.40, so the stated calculation leads to $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $600 of employee health-insurance premiums from AGI and used an incorrect $15,360 standard deduction. It also counted only $1,554 of traditional contributions for the Saver's Credit, excluding the Roth 401(k) and Roth IRA contributions; the qualified base is $1,924 and the credit is $192.40."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied an assumed TCJA sunset, replacing the 2026 $16,100 standard deduction and 10%/12% brackets with a personal exemption and 10%/15% brackets. The applicable parameters produce $20,169.67 of taxable income and $2,172.36 of tentative tax before the correctly computed $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction and used a $15,500 standard deduction rather than $16,100. Its unexplained $1,494 result also does not follow from the stated AGI and deduction; the correct tax after the $192.40 Saver's Credit is $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly introduced a personal exemption into the 2026 computation and submitted $2,576 without showing bracket arithmetic. The correct calculation uses the $16,100 standard deduction, produces $2,172.36 of tentative tax, and subtracts the $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI for health-insurance premiums, producing $35,970 instead of $36,269.67, and then applied obsolete sunset-era deductions and rates. The 2026 standard deduction and brackets yield $2,172.36 before the same $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly identified the $192.40 Saver's Credit but used the wrong 2026 standard deduction and bracket threshold, including a $15,700 deduction and $11,925 10% ceiling. The correct $16,100 deduction and 2026 rates produce $2,172.36 before the credit and $1,979.96 after it."
+us,scenario_070,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model invoked a personal exemption and unspecified 2026 parameters that inflated tentative tax to $2,780.50 before its stated $192.40 credit. The applicable $16,100 standard deduction and 10%/12% brackets instead produce tentative tax of $2,172.36."
+us,scenario_070,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly computed AGI and the $192.40 Saver's Credit but used estimated parameters: a $15,420 standard deduction and a $12,259 bracket threshold. The actual $16,100 standard deduction reduces taxable income to $20,169.67 and tentative tax to $2,172.36."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented child, dependent-care, or other nonrefundable credits even though the household has no child, dependent, or qualifying care expense. Only the $192.40 Saver's Credit offsets the $2,172.36 tentative tax, leaving $1,979.96 rather than zero."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model assumed unspecified nonrefundable credits eliminate the liability despite the instruction that unlisted facts are false and the absence of dependents or qualifying credit expenses. The available Saver's Credit is $192.40 and does not erase the $2,172.36 tentative tax."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied projected post-TCJA-sunset law, using a reduced deduction plus personal exemption and 10%/15% brackets. It also stated that no nonrefundable credit applies, omitting the $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model correctly derived taxable income of about $20,170 and tentative tax of $2,172.40 but incorrectly concluded that no nonrefundable credit was available. The taxpayer qualifies for a $192.40 Saver's Credit on $1,924 of retirement contributions, reducing the amount to $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model stopped at the correctly calculated no-credit tax of about $2,172.40. It omitted the applicable $192.40 Saver's Credit on the traditional and Roth retirement contributions."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $1,623 does not follow from the stated deductions: they yield $20,169.67 of taxable income and $2,172.36 of tentative tax. Subtracting the only applicable nonrefundable credit, the $192.40 Saver's Credit, yields $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied sunset-era law, using an $8,300 standard deduction, a $5,300 personal exemption, and a 15% second bracket. The applicable 2026 $16,100 standard deduction and 10%/12% brackets produce $2,172.36 before its correctly identified $192.40 Saver's Credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly used post-TCJA-sunset deductions, a personal exemption, and 10%/15% rates to calculate about $2,810 of tentative tax. Under the applicable 2026 parameters, tentative tax is $2,172.36 and the $192.40 Saver's Credit leaves $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted health-insurance premiums from AGI and used a $15,400 rather than $16,100 standard deduction. It also limited Saver's Credit contributions to $1,554, excluding the Roth contributions, so it used $155 instead of the correct $192.40 credit."
+us,scenario_070,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,household_unit_or_filing_status,False,"The model treated this unmarried taxpayer with no qualifying person as head of household and used the larger head-of-household deduction and brackets. The correct filing status is single, and the Saver's Credit base also includes the Roth 401(k) and Roth IRA contributions, producing a $192.40 credit on $1,924 rather than $155.40 on $1,554."
+us,scenario_070,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model stated that tentative tax was about $2,108 and the Saver's Credit about $159, then arithmetically asserted that nonrefundable credits reduced the result to zero. No other qualifying nonrefundable credits exist; the correct tentative tax is $2,172.36 and the $192.40 Saver's Credit leaves $1,979.96."
+us,scenario_070,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model correctly computed tentative tax but assigned a 50% Saver's Credit rate to AGI of $36,269.67. The applicable rate is 10%, so the credit on $1,924 is $192.40 rather than $962."
+us,scenario_070,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model omitted the $276 student-loan-interest deduction from AGI, used incorrect 2026 deduction and bracket parameters, and applied a 50% Saver's Credit rate instead of 10%. It also excluded Roth contributions from the credit base and submitted $1,213.50 despite its own arithmetic yielding $1,475.52."
+us,scenario_070,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a Credit for Other Dependents even though the household contains only the taxpayer and has no dependent. The available nonrefundable offset is the $192.40 Saver's Credit, which reduces $2,172.36 of tentative tax to $1,979.96 rather than zero."
 us,scenario_070,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model applied the Additional Child Tax Credit earned-income formula despite the household having no qualifying child, so there was no Child Tax Credit balance available to refund. Its separate EITC rationale also misstates the rule: $100 of investment income does not exceed the limit; the childless EITC is zero because $38,000 of earnings exceeds its phaseout ceiling."
 us,scenario_070,head_medicaid_eligible,minimax-m3,llm_error,thresholds_rates,False,"The model called eligibility borderline without calculating the stated household’s MAGI-to-FPL ratio. At 2.27 times FPL, the head exceeds Illinois’s 138% FPL ACA expansion limit, and age 57, low assets, and employer-sponsored insurance do not establish another Medicaid category."
 us,scenario_070,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model treated the listed $300 health insurance premium as a pre-tax FICA exclusion even though no pre-tax treatment was specified. FICA applies to the full $38,000, yielding $2,356 of Social Security tax and $551 of Medicare tax."
@@ -4211,32 +4424,34 @@ us,scenario_070,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"T
 us,scenario_070,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model conflated regular Medicare tax with the Additional Medicare Tax and set all Medicare tax to zero because wages were below the additional-tax threshold. The ordinary 1.45% Medicare tax applies from the first dollar of covered wages, adding $551 to the correctly computed $2,356 Social Security tax."
 us,scenario_070,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages from $38,000 to $37,700 by assigning unlisted pre-tax treatment to the $300 health premium. Applying 6.2% Social Security and 1.45% Medicare rates to the full wages produces $2,907."
 us,scenario_070,payroll_tax,minimax-m3,llm_error,other,False,"The model correctly derived $2,356 of Social Security tax plus $551 of Medicare tax, then replaced the exact $2,907 total with $2,970 under an invalid claim of rounding. Rounding $2,907 does not produce $2,970, and the requested annual amount is the exact derived total."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $2,425 exemption and treated its tax effect as a nonrefundable credit instead of subtracting the 2026 $2,925 allowance from Illinois base income. Its final $1,154 also does not follow from its own stated $1,869 minus $120 arithmetic."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted an estimated $2,850 personal exemption for the 2026 Illinois allowance of $2,925. Applying 4.95% to $36,269.67 minus $2,925 yields $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model cycled through several unsupported exemption and taxable-income estimates, then submitted $1,726 even though none of its displayed calculations produced that amount. It needed to use the fixed $2,925 exemption and $33,344.67 taxable income."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated the exemption at $2,850 and then abandoned its own approximately $1,655 calculation for an unsupported $1,699. The prescribed calculation is $33,344.67 at 4.95%, producing $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly added back the traditional IRA and student-loan-interest deductions and assumed the listed employer-sponsored insurance premium was a pre-tax payroll deduction. It also used the obsolete $2,425 exemption instead of $2,925; Illinois taxable income is $33,344.67."
-us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest deduction from adjusted gross income and replaced the enacted $2,925 exemption with a $2,850 estimate. Its extra adjustment for an undefined exemption phase-in has no role in this calculation."
-us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used an unsupported AGI of $35,670 and an incorrect $2,842 exemption. The trace establishes AGI of $36,269.67 and a $2,925 exemption, leaving $33,344.67 taxable."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model invented a $135 Illinois renter's credit based on 25% of rent and also used a $2,775 exemption. Rent does not generate the claimed nonrefundable credit here, and the applicable personal exemption is $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption, yet its $1,716 answer does not follow from applying 4.95% after that exemption. The correct exemption is $2,925 and the resulting tax is $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model approximated the Illinois personal exemption at $2,500 instead of using the 2026 value of $2,925. That parameter error overstated taxable income and tax."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $35,970 by treating the health-insurance premium as a pre-tax wage exclusion, despite the prompt not identifying it as pre-tax. It also used a $2,775 exemption rather than $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $2,850 personal exemption instead of the 2026 Illinois allowance of $2,925. The additional $75 exemption reduces liability by $3.71 to $1,650.56 after exact-input rounding."
-us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption instead of $2,925. Illinois taxable income is therefore $33,344.67, not $33,495."
-us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest adjustment and used an obsolete $2,425 personal exemption. Its submitted $1,626 also contradicts its own stated $1,689 computation; the correct base is $33,344.67."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly concluded that deductions and the personal exemption eliminated Illinois taxable income. They reduce AGI only to $36,269.67 and taxable income to $33,344.67, which produces $1,650.56 at 4.95%."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model treated the listed adjustments as sufficient to reduce Illinois liability to zero. After all applicable adjustments and the $2,925 exemption, $33,344.67 remains taxable at 4.95%."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an invented $1,824 personal exemption and omitted the $276 student-loan-interest deduction from AGI. The correct inputs are AGI of $36,269.67 and an exemption of $2,925."
-us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a $2,850 exemption rather than the 2026 Illinois personal exemption of $2,925. Applying the rate to the correct $33,344.67 tax base yields $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The answer implies a taxable base of about $35,899 at 4.95%, rather than the traced $33,344.67. It failed to apply the full $2,925 personal exemption to the $36,269.67 AGI."
-us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI to $35,970 and used the obsolete $2,425 exemption. With AGI of $36,269.67 and the $2,925 exemption, taxable income is $33,344.67."
-us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an approximate $2,775 personal exemption rather than the 2026 amount of $2,925. This overstated taxable income by about $150 and produced $1,658 instead of $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model explicitly denied Illinois's personal exemption and taxed the full $36,270 base. Illinois subtracts a $2,925 exemption, leaving $33,344.67 subject to the 4.95% rate."
-us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption rather than the applicable $2,925 allowance. The correct taxable income is $33,344.67 and the tax is $1,650.56."
-us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a standard deduction to Illinois income and reduced the tax base to roughly $20,696. Illinois uses the traced AGI less the $2,925 personal exemption, producing taxable income of $33,344.67."
-us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the personal exemption allowance as a dollar-for-dollar $2,800 nonrefundable tax credit and zeroed the liability. The $2,925 allowance is instead deducted from income before applying the 4.95% rate."
-us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The submitted amount implies use of the wrong personal-exemption parameter or tax base. The exact calculation uses $36,269.67 of AGI minus the $2,925 exemption, then applies 4.95% to $33,344.67."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated the Illinois personal exemption as a tax credit and then submitted $1,154, which does not follow from even its stated $1,869 minus $120 calculation. Illinois instead subtracts the $2,925 exemption from $36,269.67 of AGI before applying 4.95%."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $2,850 personal exemption instead of the 2026 amount of $2,925. It also submitted $1,659.83 rather than the $1,654.29 produced by its own stated inputs."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model cycled among several incorrect exemption and taxable-income estimates, then submitted $1,726 without a supporting calculation. The required base is $36,269.67 minus the $2,925 exemption, or $33,344.67."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated the exemption at $2,850 and then abandoned its resulting tax of about $1,655 for an unsupported $1,699. Applying the $2,925 exemption and 4.95% rate yields $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a pre-tax treatment for the listed $300 health-insurance premium and added back the IRA and student-loan-interest deductions, producing the wrong Illinois income base. Illinois uses AGI of $36,269.67 and a $2,925 personal exemption."
+us,scenario_070,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded student-loan interest from the Illinois AGI calculation even though it belongs in the federal AGI feeding the state return, and it used an estimated $2,850 exemption. Its final $1,690 also does not follow from its stated $1,668 calculation."
+us,scenario_070,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $35,670 and used an incorrect $2,842 exemption. The traced amounts are $36,269.67 of AGI and a $2,925 exemption."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model invented a $135 Illinois renter's credit based on an assumed property-tax share of rent. Rent does not generate that nonrefundable Illinois property-tax credit, and the applicable personal exemption is $2,925 rather than $2,775."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption and submitted a result inconsistent with the stated 4.95% calculation. The 2026 exemption is $2,925, leaving $33,344.67 taxable."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model approximated the personal exemption at $2,500 instead of applying the 2026 amount of $2,925. That overstated Illinois taxable income and tax."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $35,970 by treating a listed health premium as pre-tax and also used a $2,775 exemption. The calculation requires $36,269.67 of AGI less the $2,925 exemption."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly approximated AGI but used a $2,850 personal exemption instead of $2,925. This overstated taxable income by $75 and tax by $3.71."
+us,scenario_070,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption instead of the 2026 amount of $2,925. The extra $150 of taxable income explains the higher tax."
+us,scenario_070,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the $276 student-loan-interest adjustment and used a stale $2,425 exemption, then submitted $1,626 despite calculating $1,689. The correct calculation uses $36,269.67 of AGI and a $2,925 exemption."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that retirement deductions and the personal exemption eliminate Illinois taxable income. They leave $33,344.67 taxable at 4.95%, producing $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the listed adjustments as sufficient to reduce Illinois liability to zero. After those adjustments, AGI remains $36,269.67 and the $2,925 exemption leaves $33,344.67 taxable."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an invented $1,824 personal exemption and omitted the student-loan-interest adjustment from AGI. Illinois applies the $2,925 exemption to AGI of $36,269.67."
+us,scenario_070,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a $2,850 personal exemption instead of $2,925. This left taxable income $75 too high and overstated tax by $3.73 after exact-input rounding."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $1,777 implies taxable income of about $35,899, rather than the traced $33,344.67 after the personal exemption. The model failed to apply the correct AGI adjustments and $2,925 exemption."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption instead of $2,925. Applying the correct exemption reduces taxable income by another $150 and produces $1,650.56."
+us,scenario_070,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model lowered AGI to $35,970 through unsupported premium treatment and used a stale $2,425 exemption. The correct inputs are $36,269.67 of AGI and a $2,925 exemption."
+us,scenario_070,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an approximate $2,775 exemption rather than the 2026 amount of $2,925. Its taxable-income base was therefore $150 too high."
+us,scenario_070,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly asserted that Illinois provides no personal exemption and taxed the full $36,270 AGI. Illinois subtracts the $2,925 personal exemption before applying 4.95%."
+us,scenario_070,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $2,775 personal exemption instead of $2,925. The correct net income is $33,344.67, not $33,495."
+us,scenario_070,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied a standard deduction to reach taxable income near $20,696 and then submitted a number inconsistent with its own $1,024 calculation. Illinois instead subtracts only the applicable $2,925 personal exemption here, leaving $33,344.67 taxable."
+us,scenario_070,state_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model approximated the exemption at $2,900 and used rounded AGI rather than the traced values. The exact calculation uses $36,269.67 minus $2,925, then applies 4.95%."
+us,scenario_070,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model treated the $2,800 personal exemption as a dollar-for-dollar nonrefundable credit and subtracted it from tax. Illinois uses the $2,925 exemption as a deduction from income before applying the 4.95% rate."
+us,scenario_070,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The submitted $1,658.66 implies an incorrect exemption or taxable-income base. The traced base is $36,269.67 minus the $2,925 exemption, and 4.95% of the resulting $33,344.67 is $1,650.56."
 us,scenario_070,state_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model correctly found that the childless federal EITC was fully phased out but then invented a $102 Illinois EITC despite Illinois calculating its credit as a percentage of the federal amount. The age expansion changes categorical eligibility only; it does not override the federal income phaseout, so 20% of a zero federal EITC yields $0."
 us,scenario_071,head_medicaid_eligible,claude-haiku-4.5,llm_error,asset_resource,False,"The model declared $36,542 of income below New York’s aged Medicaid limits and simultaneously treated $62,291 as below a resource limit it described as $15,900, reversing both comparisons. It therefore invented an aged Medicaid pathway even though the person’s engine category is NONE."
 us,scenario_071,head_medicaid_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model invoked unspecified senior disregards to reduce $36,542 below an aged Medicaid limit of roughly $20,000 without calculating any applicable disregard. No aged pathway is established, and the engine assigns medicaid_category = NONE."
@@ -4264,12 +4479,13 @@ us,scenario_072,state_income_tax_before_refundable_credits,grok-build-0.1,llm_er
 us,scenario_072,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used approximately $12,000 of personal exemptions rather than electing the joint $40,000 Michigan standard deduction available to this birth-year cohort. The standard deduction offsets the approximately $20,166 remaining after the Social Security subtraction."
 us,scenario_072,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model applied two $6,100 personal exemptions and taxed the residual $7,966. It omitted the alternative joint $40,000 Michigan standard deduction, which reduces the household’s remaining Michigan income to zero."
 us,scenario_072,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model repeatedly varied estimated senior personal-exemption amounts and settled on $13,000 rather than applying the applicable joint $40,000 Michigan standard deduction. That deduction, together with the separate Social Security exclusion, eliminates the entire remaining Michigan income base."
-us,scenario_073,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated receipt of SSDI as automatically establishing disability-based Medicaid eligibility and assumed the reported assets satisfied that pathway. At age 57, with no SSI and no established qualifying category, the head does not enter an aged or disabled pathway; the engine assigns Medicaid category NONE."
-us,scenario_073,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the desired traditional 401(k) contribution from Medicaid MAGI, even though the household has no wages from which that elective deferral could occur. The resulting MAGI is 1.60 times FPL, not below the 138% expansion limit, and nontaxable Social Security benefits are added back for Medicaid MAGI."
-us,scenario_073,head_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the SSDI and pension income as low without applying the expansion threshold. Those amounts produce MAGI of 1.60 times FPL, above Michigan’s 138% FPL expansion limit, and no alternative Medicaid category applies."
-us,scenario_073,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,The model asserted that the head met the expansion-group income condition without computing the applicable income ratio. The head’s MAGI equals 1.60 times FPL and therefore exceeds the 138% FPL expansion threshold.
-us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model excluded the entire $22,646 of SSDI merely because it was not taxable. Medicaid MAGI adds nontaxable Social Security benefits back into income, producing MAGI of 1.60 times FPL and disqualifying the head from Michigan’s expansion group."
-us,scenario_073,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model equated having no earnings with having sufficiently low Medicaid income and ignored the SSDI and taxable pension. Those income sources yield Medicaid MAGI of 1.60 times FPL, above the expansion limit, while the head qualifies through no other category."
+us,scenario_073,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated receipt of SSDI as proof that the head qualifies through an aged/disabled Medicaid pathway. SSDI receipt does not itself establish that categorical pathway, and the head qualifies through none of Michigan’s Medicaid categories."
+us,scenario_073,head_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the desired traditional 401(k) contribution from Medicaid MAGI despite the absence of wages from which that employee deferral could be made. The resulting MAGI is 1.60 times FPL, above Michigan’s expansion-adult limit."
+us,scenario_073,head_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the head’s Medicaid-countable income as low without applying the expansion-group income threshold. The engine’s MAGI calculation is 1.60 times FPL, which exceeds the Michigan expansion-adult limit, and no alternative category applies."
+us,scenario_073,head_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that the expansion-group income condition was met, but the head’s MAGI is 1.60 times FPL and exceeds that pathway’s limit. The head therefore cannot qualify as a Michigan expansion adult, and no other category applies."
+us,scenario_073,head_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model excluded all nontaxable SSDI from Medicaid MAGI by treating MAGI as limited to taxable income. Medicaid MAGI adds nontaxable Social Security benefits, producing income of 1.60 times FPL and placing the head above the expansion-adult limit."
+us,scenario_073,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model counted only the pension and excluded SSDI because no Social Security benefit was taxable for income-tax purposes. Medicaid MAGI includes nontaxable Social Security benefits, yielding 1.60 times FPL and disqualifying the head from the expansion-adult pathway."
+us,scenario_073,head_medicaid_eligible,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model inferred Medicaid eligibility from the absence of wages and ignored the SSDI and pension income used in the Medicaid income calculation. Those sources produce MAGI of 1.60 times FPL, above the expansion-adult limit, and the head qualifies through no other category."
 us,scenario_073,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model expressly assumed that the 24-month SSDI waiting period was satisfied, even though benefit duration was unlisted and therefore could not be supplied as a qualifying fact. Full-year constancy means the listed income and status do not change during 2026; it does not establish 24 prior months of SSDI entitlement."
 us,scenario_073,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,The model converted an assumed “established disability status” into completion of Medicare's 24-month SSDI waiting period. Receipt of SSDI during the tax year does not supply the unlisted entitlement duration required for disability-based Medicare eligibility.
 us,scenario_073,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model applied the under-65 SSDI pathway as though the required 24-month waiting period had been completed. The prompt provides SSDI income but no duration of entitlement, so that qualifying step is absent."
@@ -4286,35 +4502,37 @@ us,scenario_073,head_medicare_eligible,gpt-5.6-terra,llm_error,age_disability,Fa
 us,scenario_073,head_medicare_eligible,inkling,llm_error,age_disability,False,The model invented completion of the 24-month SSDI waiting period and misread a premium field labeled as excluding Medicare Part B as evidence of Medicare enrollment. Neither the ongoing SSDI amount nor that exclusion supplies the missing eligibility duration or Medicare status.
 us,scenario_073,head_medicare_eligible,kimi-k2.6,llm_error,age_disability,False,The model asserted that PolicyEngine makes SSDI income automatically confer Medicare eligibility. It omitted the required under-65 eligibility conditions and treated an income variable as a Medicare-status indicator.
 us,scenario_073,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no payroll-taxable wages and explicitly derived payroll tax of $0, but then submitted $2,846. Its numeric output contradicts its own payroll-tax-base reasoning and final stated derivation."
-us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model treated failure of the ordinary net-income test and a negative formula allotment as producing no SNAP. It omitted Michigan TANF non-cash categorical eligibility and the minimum allotment payable to an eligible one-person household.
-us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% gross-income limit and a $2,750 resource limit as disqualifying tests. Michigan's TANF non-cash categorical eligibility applies here, and the household also passes the trace's 1.65-FPG gross test and $3,600 asset test before receiving the minimum allotment."
-us,scenario_073,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model subtracted roughly $5,165 of annual medical expenses from a single month's income, driving monthly net income below zero. The trace yields monthly net income of $1,755.47 and a negative regular allotment, after which the one-person minimum—not the $298 maximum—is paid."
-us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,The model concluded that net income above the ordinary limit and a nonpositive formula benefit require a zero award. It failed to apply Michigan TANF non-cash categorical eligibility and the minimum allotment for an eligible one-person household.
-us,scenario_073,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model double-counted or otherwise overstated medical premiums and used an erroneous one-person maximum allotment of about $587. The trace uses $1,755.47 of monthly net income and a $298 maximum, producing the applicable minimum allotment rather than $214 per month."
-us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model made the ordinary 100%-FPL net-income test dispositive. Michigan TANF non-cash categorical eligibility preserves eligibility, and the resulting negative regular allotment is replaced by the one-person minimum."
-us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,The model treated a benefit reduction exceeding the maximum allotment as a zero payment. It omitted the minimum allotment owed because the household qualifies through Michigan TANF non-cash categorical eligibility.
-us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model used the ordinary 130%-FPL gross-income ceiling as an automatic bar. It omitted Michigan's applicable 1.65-FPG gross pathway and TANF non-cash categorical eligibility, which lead to the one-person minimum allotment."
-us,scenario_073,snap,gemini-3-flash-preview,llm_error,period_annualization,False,"The model correctly identified categorical eligibility and the minimum-benefit rule but fixed the minimum at $23 for all 12 months. PolicyEngine applies $23.84 in most months and $24.37 in the adjusted months, totaling $287.68."
-us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer treats countable income as disqualifying under the ordinary income test. The household qualifies through Michigan TANF non-cash categorical eligibility and therefore receives the one-person minimum allotment.
-us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated the absence of shelter and utility deductions, and the resulting negative regular allotment, as requiring zero SNAP. It omitted categorical eligibility and the minimum allotment for an eligible one-person household."
-us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model correctly recognized the disability-related gross-test treatment but stopped when 30% of its net income exceeded the maximum allotment. Michigan TANF non-cash categorical eligibility triggers the one-person minimum instead of zero.
-us,scenario_073,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model correctly applied categorical eligibility and the minimum allotment but annualized a flat $23 monthly value. The trace uses $23.84 in most months and $24.37 in some months, producing $287.68."
-us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model equated net income above the level yielding a positive formula allotment with no benefit. It omitted the minimum allotment available through Michigan TANF non-cash categorical eligibility.
-us,scenario_073,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model inflated annual medical expenses to $11,089 and derived the wrong net income, then treated a negative regular allotment as zero. The trace produces $1,755.47 monthly net income and, through categorical eligibility, substitutes the one-person minimum allotment."
-us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model treated the ordinary one-person income limits as dispositive. It omitted Michigan TANF non-cash categorical eligibility and the minimum allotment that follows from that eligibility.
-us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required unlisted earnings, housing costs, or a separate take-up indicator to establish SNAP. The prompt assumes take-up, and the listed facts establish TANF non-cash categorical eligibility and a one-person minimum allotment."
-us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model denied qualifying disability status even though receipt of Social Security disability income supplies the relevant disabled-household fact. It also omitted Michigan TANF non-cash categorical eligibility, under which the household receives the minimum allotment."
-us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model stopped after determining that income eliminated the ordinary formula allotment. It omitted the one-person minimum benefit preserved by Michigan TANF non-cash categorical eligibility.
-us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated countable income after deductions as eliminating SNAP entirely. Under Michigan TANF non-cash categorical eligibility, the negative regular allotment is replaced by the one-person minimum."
-us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,The model used the ordinary net-income limit as an automatic disqualification. It omitted Michigan TANF non-cash categorical eligibility and the resulting minimum allotment.
-us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,The model correctly recognized the disabled-household treatment but concluded that a 30% reduction exceeding the maximum produces zero. It failed to apply categorical eligibility's minimum allotment for a one-person household.
-us,scenario_073,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated the medical deduction by aggregating overlapping premium fields and obtained monthly net income of about $1,274 instead of $1,755.47. It then omitted the categorical-eligibility minimum allotment after its regular benefit calculation fell below zero."
-us,scenario_073,snap,inkling,llm_error,period_annualization,False,"The model correctly recognized that the negative regular allotment becomes a minimum benefit, but it annualized an approximate $23 monthly amount. PolicyEngine's monthly minima are $23.84 and $24.37 across the year, totaling $287.68."
-us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model treated 30% of net income exceeding the maximum allotment as producing no payment. It omitted Michigan TANF non-cash categorical eligibility and the mandatory one-person minimum allotment.
-us,scenario_073,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted only the standard deduction and omitted the applicable medical-expense deduction, overstating monthly net income at about $1,942 instead of $1,755.47. It also failed to replace the negative regular allotment with the minimum benefit available through categorical eligibility."
-us,scenario_073,snap,minimax-m3,llm_error,asset_resource,False,"The model counted only $1,800 of the explicitly listed $3,600 in bank and stock assets and nevertheless asserted an asset-limit failure. The trace's $3,600 resource total passes the applicable test, and TANF non-cash categorical eligibility leads to the minimum allotment despite the ordinary income calculation."
-us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,The model refused to treat an SSDI recipient as disabled for SNAP and applied the ordinary 130%-FPL gross limit. The disability fact and Michigan TANF non-cash categorical eligibility preserve eligibility and produce the one-person minimum allotment.
-us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model applied a resource ceiling under which $3,600 was disqualifying. The trace's applicable asset test is satisfied at $3,600, and Michigan TANF non-cash categorical eligibility results in the one-person minimum allotment."
+us,scenario_073,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model treated failure of the ordinary net-income test and a negative formula allotment as disqualifying. It omitted TANF non-cash categorical eligibility and the minimum allotment payable to an eligible one-person household.
+us,scenario_073,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary 130% gross-income and $2,750 resource limits instead of Michigan's TANF non-cash categorical-eligibility pathway. The household passes the applicable gross and asset tests and then receives the one-person minimum allotment."
+us,scenario_073,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted roughly $5,165 of annual medical costs from a single month's income, driving monthly net income below zero. It also substituted a maximum allotment for the minimum allotment that applies after the correct $1,755.47 monthly net-income computation."
+us,scenario_073,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,The model stopped after concluding that net income exceeded the ordinary eligibility limit and that the formula allotment was exhausted. It failed to apply TANF non-cash categorical eligibility and the mandatory minimum allotment for the eligible one-person household.
+us,scenario_073,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted overlapping employer-sponsored and other health-premium fields as separate SNAP medical expenses and thereby understated net income. It also used an erroneous one-person maximum allotment of about $587 instead of $298, producing $214 per month rather than the minimum allotment."
+us,scenario_073,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model made the ordinary 100% FPL net-income test dispositive. Michigan's TANF non-cash categorical-eligibility pathway preserves eligibility, after which the exhausted formula benefit is raised to the one-person minimum allotment."
+us,scenario_073,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,The model equated a fully offset formula allotment with zero SNAP. It omitted the minimum allotment required for this categorically eligible one-person household.
+us,scenario_073,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model applied only the ordinary 130% FPL gross-income cutoff. It omitted Michigan's TANF non-cash categorical eligibility, under which the household passes the applicable 1.65-FPL gross test and receives the minimum allotment."
+us,scenario_073,snap,gemini-3-flash-preview,llm_error,period_annualization,False,"The model identified categorical eligibility and the minimum-benefit rule but rounded every month to $23. PolicyEngine uses monthly minimums of $23.84 and, after the guideline adjustment, $24.37, which annualize to $287.68."
+us,scenario_073,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer treats excess countable income as disqualifying. The household qualifies through TANF non-cash categorical eligibility and receives the one-person minimum allotment despite the exhausted formula benefit.
+us,scenario_073,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model correctly recognized that the ordinary benefit formula is exhausted but incorrectly converted that result to zero. TANF non-cash categorical eligibility triggers the minimum allotment for this eligible one-person household even without shelter deductions.
+us,scenario_073,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model treated the fact that 30% of net income exceeds the maximum allotment as producing no benefit. For this categorically eligible one-person household, that calculation instead triggers the statutory minimum allotment."
+us,scenario_073,snap,gemini-3.6-flash,llm_error,period_annualization,False,"The model correctly applied categorical eligibility and the minimum-allotment rule but used a flat $23 for all 12 months. The applicable monthly values are $23.84 and $24.37 across the year, totaling $287.68."
+us,scenario_073,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model treated net income above the positive-benefit range as eliminating SNAP altogether. It omitted the minimum allotment payable through the household's TANF non-cash categorical eligibility.
+us,scenario_073,snap,glm-5.2,llm_error,categorical_eligibility,False,"After computing an exhausted regular allotment, the model set the benefit to zero. Because the one-person household remains categorically eligible, the exhausted formula amount is replaced by the minimum allotment."
+us,scenario_073,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model used the ordinary one-person income limits as a complete eligibility rule. It omitted TANF non-cash categorical eligibility and the resulting minimum allotment.
+us,scenario_073,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required additional earnings, housing-cost, or take-up facts to establish SNAP. The prompt directs program take-up, and the listed income and assets establish TANF non-cash categorical eligibility followed by the one-person minimum allotment."
+us,scenario_073,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model ignored that receipt of Social Security disability income establishes disabled status for the SNAP computation and then applied the ordinary gross-income test as dispositive. It also omitted TANF non-cash categorical eligibility and the minimum allotment.
+us,scenario_073,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model treated income high enough to exhaust the regular formula allotment as producing zero. The household remains eligible through the TANF non-cash pathway and therefore receives the one-person minimum allotment.
+us,scenario_073,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model correctly recognized disability and an exhausted regular allotment but stopped at zero. It failed to apply the minimum allotment required for this categorically eligible one-person household.
+us,scenario_073,snap,grok-4.3,llm_error,categorical_eligibility,False,The model made the ordinary SNAP net-income limit dispositive. TANF non-cash categorical eligibility preserves eligibility and yields the minimum allotment.
+us,scenario_073,snap,grok-4.5,llm_error,categorical_eligibility,False,The model treated the 30% net-income reduction fully offsetting the maximum allotment as a zero-benefit result. The minimum-allotment rule replaces that exhausted amount for an eligible one-person household.
+us,scenario_073,snap,grok-4.6,llm_error,categorical_eligibility,False,The model correctly found that the regular formula produces no positive allotment but incorrectly stopped there. Michigan TANF non-cash categorical eligibility requires payment of the one-person minimum allotment.
+us,scenario_073,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model concluded that a benefit reduction larger than the maximum allotment forces payment to zero. It omitted the minimum allotment that applies because the household is categorically eligible.
+us,scenario_073,snap,inkling,llm_error,period_annualization,False,The model correctly identified the minimum-allotment rule but rounded it to a flat $23 per month. Applying the actual monthly minimums of $23.84 and $24.37 produces $287.68 for the year.
+us,scenario_073,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model equated an exhausted formula allotment with loss of all SNAP. TANF non-cash categorical eligibility keeps the household eligible and activates the one-person minimum allotment.
+us,scenario_073,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted only the standard deduction and omitted the applicable medical deduction, overstating monthly net income relative to $1,755.47. More importantly, it treated the exhausted formula amount as zero instead of applying the minimum allotment through categorical eligibility."
+us,scenario_073,snap,minimax-m3,llm_error,asset_resource,False,"The model misstated both the countable resources and their effect: bank and stock assets total $3,600, not $1,800, and the household passes the applicable asset test. It also omitted TANF non-cash categorical eligibility, which leads to the minimum allotment."
+us,scenario_073,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model denied any applicable medical deduction despite the disability income and listed health expenses, overstating net income. It then failed to apply TANF non-cash categorical eligibility and the one-person minimum allotment."
+us,scenario_073,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,The model rejected SSDI as establishing disability for SNAP and applied only an incorrectly stated ordinary gross-income limit. The household qualifies through Michigan's TANF non-cash categorical-eligibility pathway and receives the minimum allotment.
+us,scenario_073,snap,qwen3.8-max,llm_error,asset_resource,False,"The model treated $3,600 of bank and stock assets as exceeding the applicable SNAP resource limit. The household passes the asset test and qualifies through TANF non-cash categorical eligibility, producing the one-person minimum allotment."
 us,scenario_074,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_074,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_074,head_medicaid_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model understated Medicaid MAGI by counting only the IRA distribution plus a taxable portion of Social Security and pension income. The engine’s MAGI calculation is 2.13 times FPL, which exceeds Louisiana’s 138% FPL expansion-adult limit and leaves no qualifying Medicaid category."
@@ -4322,48 +4540,50 @@ us,scenario_074,head_medicaid_eligible,claude-sonnet-4.6,llm_error,health_covera
 us,scenario_074,head_medicaid_eligible,claude-sonnet-5,llm_error,health_coverage,False,"The model treated receipt of SSDI as sufficient for a disability-based Medicaid pathway and also understated MAGI by broadly excluding Social Security benefits. SSDI does not itself confer SSI-related Medicaid eligibility, the person receives no SSI, and engine MAGI of 2.13 times FPL exceeds the expansion-adult limit."
 us,scenario_074,head_medicaid_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model labeled the reported sources as producing very low countable income without applying the engine’s MAGI calculation. That calculation yields 2.13 times FPL, above Louisiana’s expansion-adult limit, and no other eligibility category applies."
 us,scenario_074,head_medicaid_eligible,gpt-5.6-terra,llm_error,health_coverage,False,"The model asserted that taxable income was within the adult Medicaid limit, but the relevant engine MAGI equals 2.13 times FPL. This exceeds the 138% FPL expansion threshold, and the person qualifies through no alternative Medicaid pathway."
-us,scenario_074,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted current SSDI receipt into completion of the 24-month Medicare waiting period. Constant status during 2026 does not establish SSDI entitlement before 2026, so this 54-year-old has no stated Medicare eligibility pathway."
-us,scenario_074,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model explicitly assumed that already receiving SSDI means the 24-month waiting period has been met. SSDI receipt for the stated year supplies no prior entitlement date, and at age 54 the head does not qualify through the age pathway."
-us,scenario_074,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model treated constant year-long SSDI status as proof of the 24 months of disability entitlement required for Medicare. A single full year does not satisfy or establish that waiting period, leaving the 54-year-old ineligible."
-us,scenario_074,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model inferred completion of the 24-month SSDI waiting period from disability income being constant throughout 2026. The prompt provides no pre-2026 entitlement history, and age 54 does not meet Medicare's age threshold."
-us,scenario_074,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated SSDI benefits as immediately conferring Medicare eligibility under PolicyEngine's disability rule. SSDI receipt alone does not establish the required entitlement duration, while the head is below age 65."
-us,scenario_074,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model expressly inferred that SSDI was ongoing before the stated year and therefore that the 24-month waiting period had elapsed. The prompt prohibits inferring unlisted benefit history; constant status throughout 2026 establishes only that year's receipt, and the head is 54."
-us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,The model treated being an SSDI recipient as sufficient for Medicare eligibility. It omitted the disability-entitlement waiting period and the absence of any stated history satisfying it; the head is also under 65.
-us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model equated Social Security disability income receipt with Medicare eligibility. Current receipt does not establish completion of the required waiting period, and the 54-year-old does not qualify by age."
-us,scenario_074,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model treated SSDI income as automatically qualifying the head for Medicare. It failed to require established completion of the disability-entitlement waiting period, and age 54 is below the ordinary eligibility threshold."
-us,scenario_074,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model incorrectly made SSDI receipt itself a sufficient Medicare eligibility condition. No qualifying prior entitlement duration is listed, and the head is only 54."
-us,scenario_074,head_medicare_eligible,gemini-3.7-flash,llm_error,age_disability,False,"The model equated SSDI receipt with immediate Medicare eligibility. It skipped the required waiting-period history, which cannot be inferred from the annual income fact, and the head does not meet the age-65 rule."
-us,scenario_074,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,The model asserted that PolicyEngine treats every SSDI recipient as Medicare eligible regardless of age. It omitted the required duration of disability entitlement and supplied no stated basis for bypassing the age-65 threshold.
-us,scenario_074,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model correctly rejected age-based eligibility but then treated disability income as establishing the alternative pathway. SSDI income alone does not prove completion of the Medicare waiting period, and unlisted entitlement history must remain false."
-us,scenario_074,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated receipt of SSDI during the year as sufficient for disability-based Medicare eligibility. It failed to establish the required prior entitlement period, so the under-65 head remains ineligible."
-us,scenario_074,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model claimed that SSDI receipt for the full year satisfies a 24-month waiting period. One year is 12 months, and the prompt supplies no earlier entitlement period; the head is also below age 65."
-us,scenario_074,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model cited the statutory waiting period but treated current SSDI receipt as proof that it had elapsed. No SSDI entitlement start date or prior-period status is listed, so the disability pathway is not established for this 54-year-old."
-us,scenario_074,head_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model explicitly assumed the 24-month waiting period was complete because the head received ongoing disability benefits. That assumption invents unlisted entitlement history prohibited by the prompt, while age 54 fails the age pathway."
-us,scenario_074,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model treated Social Security disability income as a sufficient disability-based Medicare qualifier. It omitted the required waiting-period history, which is not supplied by the annual SSDI amount, and the head is under 65."
-us,scenario_074,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,The model inferred completion of the SSDI waiting period from the instruction that facts remain constant throughout the tax-benefit year. That instruction covers only 2026 and does not create the preceding entitlement history required for Medicare; the head is 54.
+us,scenario_074,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model inferred both qualifying disability entitlement and completion of Medicare's 24-month waiting period from SSDI income alone. Constant annual receipt does not establish either unlisted fact, and age 54 fails the age-based eligibility test."
+us,scenario_074,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model treated current SSDI receipt as proof that the 24-month Medicare waiting period had already elapsed. The prompt supplies no entitlement start date or qualifying Medicare disability status, so those unlisted facts are false and the 54-year-old is not eligible."
+us,scenario_074,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model converted constant year-long SSDI receipt into completion of the 24-month waiting period. A single full tax year does not establish 24 months of entitlement, and the head does not meet the age-65 rule."
+us,scenario_074,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model inferred Medicare disability eligibility and a completed 24-month waiting period from annual SSDI income. The constant-status instruction does not supply prior entitlement duration, while the stated age of 54 fails the applicable age test."
+us,scenario_074,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model incorrectly asserted that SSDI income itself triggers PolicyEngine's disability-based Medicare rule. No qualifying Medicare disability-entitlement input is listed, so it defaults to false, leaving the 54-year-old ineligible."
+us,scenario_074,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model assumed that no SSDI start date meant ongoing receipt long enough to complete the 24-month waiting period. The prompt instead requires unlisted facts to be false, so no completed waiting period is established and age 54 does not qualify."
+us,scenario_074,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model treated receipt of Social Security disability income as sufficient Medicare eligibility. Income receipt does not establish the required under-65 Medicare entitlement facts, and the head is only 54."
+us,scenario_074,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model equated Social Security disability income with Medicare eligibility without applying the separate eligibility test. With no qualifying disability-entitlement status supplied, the 54-year-old fails the age-based rule."
+us,scenario_074,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model used SSDI income as an automatic Medicare qualifier. The prompt does not establish the separate under-65 disability-entitlement pathway, and age 54 is below the Medicare age threshold."
+us,scenario_074,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model incorrectly made Medicare eligibility follow directly from Social Security disability income. PolicyEngine does not infer the missing disability-entitlement conditions from that income amount, so the age-54 head is ineligible."
+us,scenario_074,head_medicare_eligible,gemini-3.7-flash,llm_error,age_disability,False,"The model treated SSDI receipt as conclusive evidence of Medicare entitlement. The required under-65 entitlement status is unlisted and therefore false, while the head does not satisfy the age-65 test."
+us,scenario_074,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,"The model misstated PolicyEngine's rule as making every SSDI recipient Medicare eligible regardless of age. SSDI income is not itself the qualifying Medicare status, and the head's age of 54 yields no age-based eligibility."
+us,scenario_074,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model acknowledged that age 54 does not qualify but then inferred the disability pathway from disability-related income. The prompt forbids inferring unlisted status inputs, so the missing Medicare disability-entitlement condition remains false."
+us,scenario_074,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated annual Social Security disability income as sufficient for the under-65 Medicare pathway. The income amount does not supply the required Medicare entitlement status or waiting-period completion, and the head is below age 65."
+us,scenario_074,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model claimed that full-year SSDI receipt satisfies Medicare's 24-month waiting period. One year of constant receipt does not establish two years of entitlement, and no prior duration is listed."
+us,scenario_074,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model invoked the statutory waiting-period pathway without establishing that the waiting period had elapsed. SSDI income alone supplies neither entitlement duration nor PolicyEngine's qualifying disability status, leaving the 54-year-old ineligible."
+us,scenario_074,head_medicare_eligible,grok-4.6,llm_error,age_disability,False,"The model assumed that full-year SSDI receipt automatically makes an under-65 beneficiary Medicare eligible. The separate qualifying entitlement facts are absent and default to false, so age 54 controls the result."
+us,scenario_074,head_medicare_eligible,grok-build-0.1,llm_error,age_disability,False,"The model explicitly assumed the 24-month waiting period was met because disability benefits were ongoing. The prompt requires unlisted duration and status facts to be false, so that assumption is invalid and the age-54 head does not qualify."
+us,scenario_074,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model treated Social Security disability income as a qualifying Medicare disability status. PolicyEngine does not derive that status solely from the income field, and the head fails the age threshold at 54."
+us,scenario_074,head_medicare_eligible,qwen3.8-max,llm_error,age_disability,False,"The model used constant full-year SSDI receipt to infer satisfaction of Medicare's required waiting period. Constancy within 2026 does not establish 24 months of prior entitlement, and the prompt supplies no separate qualifying disability-entitlement fact."
 us,scenario_074,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,The model applied employee payroll-tax rates to non-wage income despite acknowledging that the household's income came from Social Security disability. It failed to set unlisted wages to zero and therefore invented a covered-earnings base that the household facts do not contain.
 us,scenario_074,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, violating the required output contract."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s written calculation reaches the correct $14,777 total and then submits $12,523 without any supporting computation. Its numeric output contradicts its own ordinary-income tax of $13,843 plus $934 of preferential-rate tax."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,age_disability,False,"The model treated a 62-year-old as qualifying for an age-65 additional standard deduction and then invented a $5,252 disability adjustment. Disability alone does not create either deduction, so the applicable standard deduction is $16,100 and no such disability subtraction reduces the tax."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated bracket cutoffs of $12,150 and $49,400 instead of the applicable 2026 cutoffs reflected in the trace. This overstated ordinary tax as $13,948 rather than $13,843.28; the $933.86 preferential-rate component was otherwise handled correctly."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model initially used a $15,000 standard deduction and then substituted projected bracket thresholds, rather than applying the enacted 2026 $16,100 deduction and applicable rate schedule. Those errors raised ordinary taxable income and produced $14,153.20 of ordinary tax instead of $13,843.28."
-us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model explicitly derived the correct components—$13,843.20 of ordinary tax and about $933.90 of capital-gains tax—then discarded that result and submitted $6,710 after an unexplained recalculation. Nothing in the household facts or federal calculation supports that reduction."
-us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the pre-TCJA structure returned in 2026, using an $8,600 standard deduction, a personal exemption, and 10%/15%/25% brackets. The applicable calculation instead uses a $16,100 standard deduction, no personal exemption, and the 10%/12%/22% ordinary brackets at this income."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model combined an understated $13,600 deduction with obsolete 10%/15%/25% brackets. Applying the $16,100 standard deduction and the applicable 10%/12%/22% schedule yields $13,843.28 of ordinary tax, not $17,315."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $14,930.50 exceeds the trace result after the correct $16,100 standard deduction and preferential-rate stacking. The answer implies different deduction or bracket parameters; the correct components are $13,843.28 of ordinary tax and $933.86 of capital-gains tax."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model assumed expiration of the TCJA individual provisions and reinstatement of personal exemptions. The applicable 2026 computation retains the $16,100 single standard deduction, no personal exemption, and the 10%/12%/22% ordinary rate schedule at this income."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset $8,300 standard deduction, a $5,150 personal exemption, and reverted 10%/15%/25% brackets. The applicable rules use a $16,100 standard deduction, no personal exemption, and produce $13,843.28 of ordinary tax before adding $933.86 on preferential income."
-us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used an $8,300 standard deduction, a $5,050 personal exemption, and post-sunset ordinary brackets. Those are not the applicable 2026 parameters; the correct ordinary taxable income is $86,960.37 and its tax is $13,843.28."
-us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,400 standard deduction instead of $16,100, overstating taxable income by $700. It also used estimated bracket thresholds, causing ordinary tax of $14,060.20 rather than $13,843.28."
-us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model’s $9,468 answer omits $5,309.15 of the tax generated by the applicable ordinary brackets and preferential-rate stacking. With $86,960.37 of ordinary taxable income, ordinary tax alone is $13,843.28, before the additional $933.86 capital-gains tax."
-us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated $600 of medical spending as supporting itemized deductions that drastically reduce tax. Medical expenses are deductible only above the applicable AGI floor and itemizing would not exceed the $16,100 standard deduction, so these expenses do not reduce taxable income."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model’s unexplained $13,361 total is below the $13,843.28 tax on ordinary income alone. Correct preferential-rate stacking then adds $933.86, producing $14,777.15 rather than the submitted amount."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a TCJA sunset, using an approximately $8,344 standard deduction, a personal exemption, and 10%/15%/25% brackets. The applicable rules instead use the $16,100 standard deduction, no personal exemption, and the 10%/12%/22% ordinary brackets."
-us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset $8,522 standard deduction and obsolete 10%/15%/25% brackets, while also omitting the personal exemption its assumed sunset regime would entail. Under the applicable rules, taxable income is $93,186.13 and ordinary tax is $13,843.28, not $17,916."
-us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. This is a missing-output contract failure rather than a tax computation.
-us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented roughly $18,608 of nonrefundable senior or disability credits and subtracted them from its tax. At age 62 with AGI of $109,286.13 and $90,020 of taxable pension income, no such credit offsets the liability; the trace contains no nonrefundable credit reduction."
-us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $15,750 standard deduction rather than the applicable $16,100 deduction and relied on 2025-style bracket thresholds. That overstated ordinary taxable income by $350 and produced $14,122.20 of ordinary tax instead of $13,843.28."
-us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model wrongly included the $2,240 tax-exempt pension in AGI, incorrectly treated the 62-year-old as over 65, and misstated the income sum as $131,866. It then contradicted its own statement that the elderly-or-disabled credit was zero by reducing a claimed $20,146 tax liability to zero."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s written calculation reaches approximately $14,777 using the correct deduction, brackets, and capital-gain treatment, but it submits $12,523 instead. The submitted value is an unsupported final-answer substitution that contradicts its own completed computation."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated age 62 as qualifying for an age-65 additional standard deduction and invented both a $9,000 age/blindness addition and a $5,252 disability adjustment. Disability by itself does not generate either deduction, so the applicable standard deduction is $16,100 and no such adjustment reduces the tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated 2026 single bracket thresholds of $12,150 and $49,400 instead of the applicable $12,400 and $50,400 thresholds. That overstates ordinary-income tax; the correct ordinary tax is $13,843.28, followed by $933.86 of preferential tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction and then improvised projected bracket thresholds. The applicable deduction is $16,100, leaving $86,960.37 of ordinary taxable income taxed under the $12,400 and $50,400 bracket thresholds."
+us,scenario_075,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived ordinary tax of about $13,843.20 and preferential tax of about $933.90, totaling about $14,777.10, then discarded that result and submitted $6,710 without any computation. Its final value directly contradicts its own correct derivation."
+us,scenario_075,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions had expired, replacing the $16,100 standard deduction and 10%/12%/22% brackets with an $8,600 deduction, a personal exemption, and 10%/15%/25% brackets. The 2026 computation uses the retained $16,100 deduction and current-law bracket structure."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset deduction-and-exemption regime and 10%/15%/25% ordinary brackets. The applicable 2026 rules provide a $16,100 standard deduction and tax the ordinary portion under the 10%/12%/22% schedule."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $14,930.50 implies that the model did not use the applicable combination of the $16,100 standard deduction and 2026 single thresholds. Those inputs produce $13,843.28 of ordinary tax plus $933.86 of preferential tax, not $14,930.50."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model expressly assumed expiration of the TCJA standard deduction and reinstatement of personal exemptions. The applicable 2026 law instead uses the $16,100 single standard deduction and the retained 10%/12%/22% ordinary brackets."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly reverted to an $8,300 standard deduction, a $5,150 personal exemption, and 10%/15%/25% brackets under a presumed TCJA sunset. The correct calculation uses a $16,100 standard deduction and the 2026 10%/12%/22% single schedule."
+us,scenario_075,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset regime with an $8,300 deduction, a $5,050 exemption, and older 10%/15%/25% rates. The applicable rules use the $16,100 standard deduction and 10%/12%/22% brackets for this ordinary income."
+us,scenario_075,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,400 standard deduction rather than the applicable $16,100 amount. This left ordinary taxable income $700 too high and produced the corresponding excess ordinary-income tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The submitted $9,468 does not result from applying the 2026 single schedule to $86,960.37 of ordinary taxable income and stacking $6,225.76 of preferential income above it. Those steps yield $13,843.28 of ordinary tax and $933.86 of capital-gain tax."
+us,scenario_075,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the $600 of listed medical spending as producing material itemized deductions and reduced tax to $1,446. Medical expenses are deductible only above the 7.5%-of-AGI floor, so these expenses produce no medical itemized deduction and the $16,100 standard deduction applies."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The submitted $13,361 understates the result of the applicable rate calculation. Taxing $86,960.37 of ordinary income yields $13,843.28 before adding the separate $933.86 tax on stacked long-term gains and qualified dividends."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, substituted a small standard deduction plus personal exemption, and used 10%/15%/25% brackets. The correct 2026 treatment uses the $16,100 standard deduction with 10%/12%/22% ordinary brackets."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly used a post-TCJA $8,300 deduction, $5,300 personal exemption, and 10%/15%/25% rate schedule. The applicable calculation uses the $16,100 standard deduction and retained 10%/12%/22% brackets."
+us,scenario_075,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model assumed only an $8,522 standard deduction and omitted the applicable $16,100 deduction, leaving taxable income overstated by $7,578. It also applied obsolete 15% and 25% ordinary brackets instead of the 12% and 22% brackets."
+us,scenario_075,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. Its response therefore failed the required structured-output contract rather than performing a substantive tax calculation.
+us,scenario_075,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented approximately $18,608 of nonrefundable elderly or disability credits to reduce its calculated tax to $1,003. The Credit for the Elderly or Disabled is eliminated at this income and taxable pension level, so no such credit offsets the liability."
+us,scenario_075,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used a $15,750 standard deduction instead of $16,100 and used 2025-style bracket thresholds of $11,925 and $48,475. The correct deduction and 2026 thresholds reduce ordinary tax to $13,843.28, with $933.86 added for preferential income."
+us,scenario_075,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model’s zero submission contradicts its own statement that tax before credits is $20,146 and that the elderly or disabled credit provides no usable offset. It also incorrectly included tax-exempt pension income, treated the 62-year-old as over 65, and made an arithmetic error in stating AGI as $131,866."
 us,scenario_075,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_075,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the listed disability as sufficient to activate a Florida disabled-adult Medicaid pathway without applying that pathway's complete eligibility requirements. The head receives no SSI and the engine assigns medicaid_category NONE, so no categorical pathway exists and Medicaid eligibility is 0."
 us,scenario_075,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model equated the generic disability flag with disability-based Medicare entitlement. It skipped the requirement for qualifying SSDI-based entitlement and its waiting period, while the head is only 62."
@@ -4382,38 +4602,41 @@ us,scenario_075,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The m
 us,scenario_075,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
 us,scenario_075,payroll_tax,minimax-m3,llm_error,other,False,"The model's own component calculation produced $968.18 before engine-level precision, but it then changed that result to $989 under the label of rounding. Rounding cannot turn approximately $968.2 into $989; the PolicyEngine components total $968.21."
 us,scenario_076,child2_wic_eligible,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model falsely extended Idaho WIC eligibility through age 8 even though WIC child eligibility ends at the fifth birthday. It also divided or expressed the income ratio incorrectly: using its own $23,030 poverty guideline, $173,820 is about 755% of poverty and exceeds the 185% limit by a wide margin; medical expenses do not establish WIC eligibility or cure either failure."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's stated bracket calculation produced about $24,321 of tentative tax, but it abandoned that result and substituted an unexplained $29,226 tentative-tax figure. Applying the $4,400 CTC and $200 CDCC to the correct $24,211.80 tentative tax yields $19,611.80."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,600 of child support in gross income, deducted the $6,480 employer-sponsored premium from AGI, omitted the $800 QBI deduction, and used an inflated standard deduction. It also used only $4,000 of CTC and omitted the $200 CDCC instead of subtracting $4,400 and $200."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest and state-tax itemized deductions from a mortgage balance and an estimated tax liability, even though no mortgage interest or deductible tax payment was listed. The benchmark therefore uses the $24,150 standard deduction, not its estimated $37,900 itemized deduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest and SALT deductions from the mortgage balance and an estimated state tax, then contradicted its own $14,306 calculation by submitting $21,625. The correct deduction is the $24,150 standard deduction, followed by the $800 QBI deduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated only about $1,000 of the $4,400 CTC as a usable nonrefundable credit because part of the CTC can be refundable. The $24,211.80 liability is sufficient to use the entire $4,400 CTC nonrefundable, followed by the $200 CDCC."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model used single-filer brackets and fabricated $37,300 of itemized deductions from assumed mortgage interest and SALT. The taxpayer qualifies as head of household and takes the $24,150 standard deduction, producing $139,270 of taxable income after QBI."
-us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used the single standard deduction and single tax brackets despite identifying a head-of-household taxpayer. It also subtracted the listed employer-sponsored premium from wages even though the annual gross wage input enters AGI at $160,000."
-us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced gross wages by the $6,480 employer-sponsored insurance premium and used a $23,200 standard deduction instead of $24,150. AGI is $164,220, and taxable income after the standard and QBI deductions is $139,270."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a supposed TCJA sunset regime with personal exemptions, a reduced standard deduction, and a $1,000-per-child CTC. For 2026 the applicable computation uses the $24,150 head-of-household standard deduction, no personal exemptions, and a $2,200 credit for each child."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an incorrect deduction or tax-bracket computation: AGI of $164,220 less the $24,150 standard deduction and $800 QBI deduction produces $139,270 of taxable income and $24,211.80 of tentative tax. Subtracting the $4,400 CTC and $200 CDCC gives $19,611.80, not $18,985."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied an obsolete post-sunset structure with personal exemptions, a roughly $12,090 standard deduction, and pre-TCJA brackets. It also failed to subtract the $4,400 CTC; the 2026 computation uses the $24,150 standard deduction and subtracts both CTC and CDCC."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model wrongly declared the CTC fully phased out at $157,740 of its calculated AGI and applied obsolete sunset deductions and exemptions. The household is below the $200,000 head-of-household phaseout threshold and receives the full $4,400 CTC."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model reduced gross wages by the employer-sponsored premium and did not follow the traced deduction-and-credit sequence. Gross wages remain $160,000, producing $164,220 of AGI, $139,270 of taxable income, and $19,611.80 after the $4,600 of nonrefundable credits."
-us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model incorrectly phased out the CTC at an AGI it computed as $157,740. The applicable phaseout begins at $200,000 for this head-of-household filer, so the full $4,400 CTC is subtracted along with the $200 CDCC."
-us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $6,480 employer-sponsored premium from gross wages and used a $23,100 standard deduction. The correct AGI is $164,220 and the correct standard deduction is $24,150; the CTC is also $4,400 rather than $4,000."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $14,143 is inconsistent with the traced calculation from $164,220 of AGI to $139,270 of taxable income and $24,211.80 of tentative tax. Even after the full $4,400 CTC and $200 CDCC, liability is $19,611.80, so the answer implies an excessive unlisted deduction or credit."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the $9,600 of child support as taxable income. Child support is excluded, leaving AGI of $164,220, and the model also failed to implement the traced standard deduction, QBI deduction, and $4,600 of nonrefundable credits correctly."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the $160,000 gross-wage input by the $6,480 employer-sponsored insurance premium and relied on estimated deductions and brackets. It also used a $4,000 CTC instead of the applicable $4,400."
-us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status and a $32,200 standard deduction despite no spouse being listed. The taxpayer files as head of household and uses the $24,150 standard deduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unexplained $23,250 does not follow the required sequence: $139,270 of taxable income generates $24,211.80 of tentative tax, from which $4,400 of CTC and $200 of CDCC are subtracted. The submitted amount therefore omits or miscomputes the nonrefundable-credit reduction."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an estimated $23,063 standard deduction and estimated brackets, producing $24,570 of tentative tax instead of $24,211.80. The correct standard deduction is $24,150, and the CTC is $4,400 rather than $4,000."
-us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly invoked a TCJA-expiration regime with personal exemptions, a $12,585 standard deduction, and inflation-adjusted 2017 brackets. It also reduced gross wages by the ESI premium and omitted the $4,400 CTC."
-us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model subtracted only about $1,000 of nonrefundable CTC instead of the full $4,400 available for the two qualifying children. The tentative tax is high enough to absorb the entire credit, and the additional $200 CDCC is also used."
-us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. Its response therefore failed the required submission contract before any substantive tax calculation could be evaluated.
-us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived AGI, deductions, taxable income, and both credits, but computed tentative tax as $24,332.80 instead of $24,211.80. Applying the 2026 head-of-household brackets to $139,270 yields $24,211.80, leaving $19,611.80 after credits."
-us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented $1,815 of rental depreciation even though unlisted expenses must be zero, reducing both rental income and the QBI deduction. Rental income remains $4,000, AGI is $164,220, and the QBI deduction is $800; it also used a $4,000 CTC instead of $4,400."
-us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly brackets and a purported $16,600 MFJ standard deduction despite no spouse being present. The correct filing status is head of household, with a $24,150 standard deduction and a $4,400 rather than $4,000 CTC."
-us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included nontaxable child support, subtracted the ESI premium, invented dependent deductions, and then used tax brackets and arithmetic that conflict with its own stated taxable income. It also incorrectly declared the CTC phased out using a $400,000 threshold even though its claimed AGI was far below that figure; the full $4,400 CTC and $200 CDCC apply."
-us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated the ACTC income phase-in and per-child cap as creating a refundable payment without first determining whether any CTC remained after offsetting federal income tax liability. The household’s liability absorbs the entire CTC nonrefundably, so neither its unsupported $2,400 adjustment nor any other refundable CTC remains."
-us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,other,False,"The model automatically assigned the $1,700-per-child refundable cap to both children. That cap only limits an otherwise available refundable CTC; because the full CTC is used against this household’s substantial federal income tax liability, the refundable remainder is zero."
-us,scenario_076,federal_refundable_credits,inkling,llm_error,other,False,"The model equated satisfying the ACTC earned-income phase-in with receiving the maximum refundable amount for each child. It omitted the prior step allocating the CTC against federal income tax liability, which uses the full credit nonrefundably and leaves zero refundable CTC."
-us,scenario_076,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required structured-output contract."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model's stated bracket calculation produced about $24,321 of tentative tax, but it abandoned that result and substituted an unexplained $29,226 tentative tax. Applying the $4,400 CTC and $200 CDCC to the correct $24,211.80 tentative tax yields $19,611.80."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $9,600 of child support in gross income, deducted $6,480 of employer-sponsored premiums from wages, omitted the $800 QBI deduction, and used invented deduction and bracket parameters. It also used only $4,000 of CTC and omitted the $200 CDCC instead of subtracting $4,600 of nonrefundable credits."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model fabricated mortgage interest from the mortgage balance and an assumed 7% rate and fabricated a SALT deduction, even though neither mortgage interest paid nor deductible state tax was supplied. The standard deduction is $24,150, and the model's final $21,889 also contradicts its own computed $17,033."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest and SALT itemized deductions from a balance and assumed rates rather than using the $24,150 standard deduction. Its submitted $21,625 also discards its own $14,306 calculation without a valid computation."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated only about $1,000 of the $4,400 CTC as reducing pre-refund tax by confusing the refundable-credit limit with the amount usable as a nonrefundable credit. The full $4,400 CTC and $200 CDCC reduce the $24,211.80 tentative tax to $19,611.80."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model used single filing status and fabricated itemized mortgage-interest and SALT deductions, despite correctly identifying a head-of-household taxpayer and having no mortgage-interest-paid input. It also used a $4,000 CTC instead of the 2026 $4,400 amount."
+us,scenario_076,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model called the taxpayer head of household but applied the single standard deduction and single brackets. It also deducted the listed employer premium from wages without an input establishing pre-tax wage exclusion and used a $4,000 rather than $4,400 CTC."
+us,scenario_076,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the $6,480 employer-sponsored insurance premium and used a $23,200 rather than $24,150 head-of-household standard deduction. It also omitted the $800 QBI deduction and understated the CTC by $400."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-expiration regime with personal exemptions, a reduced standard deduction, and a reverted $1,000-per-child CTC. For 2026 the applicable computation uses a $24,150 standard deduction, no personal exemptions, an $800 QBI deduction, and a $2,200 credit for each child."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The answer implies use of only $4,000 of CTC and omits the $200 CDCC and $800 QBI deduction identified in the trace. The correct credits are $4,400 of CTC plus $200 of CDCC after computing $24,211.80 of tax on $139,270."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied an obsolete post-sunset tax regime with personal exemptions and pre-TCJA brackets instead of the governing 2026 parameters. It also deducted the employer premium from wages and failed to apply the $4,400 CTC and $800 QBI deduction."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model wrongly declared the CTC fully phased out at $157,740 of its calculated AGI. The head-of-household CTC phaseout starts at $200,000, so both children generate the full $4,400 credit; it also used obsolete sunset deductions and improperly reduced wages by the ESI premium."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer's reference to income after pre-tax health-insurance deductions shows that it subtracted the $6,480 premium from wages without an input establishing that treatment. The correct AGI is $164,220, followed by the $24,150 standard deduction, $800 QBI deduction, and $4,600 of nonrefundable credits."
+us,scenario_076,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model wrongly treated the CTC as phased out at AGI of $157,740. The phaseout threshold is $200,000 for this filer, so the full $4,400 CTC applies, and the listed ESI premium does not reduce the $160,000 wage input."
+us,scenario_076,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $6,480 of employer-sponsored premiums from the reported wages and used an incorrect $23,100 standard deduction. It also used a $4,000 CTC rather than $4,400, producing the wrong AGI, taxable income, and final liability."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The unexplained $14,143 result does not follow from the stated exclusions and child credits: $164,220 of AGI less the $24,150 standard deduction and $800 QBI deduction produces $24,211.80 of tentative tax. Subtracting $4,400 of CTC and $200 of CDCC yields $19,611.80, so the answer embeds an unsupported additional reduction."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the $9,600 child-support receipt as taxable income, although child support is excluded from federal gross income. It also failed to apply the specified $24,150 standard deduction, $800 QBI deduction, and full $4,600 of nonrefundable credits coherently."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by $6,480 for the ESI premium, lowering AGI from $164,220 to $157,740. It also used an estimated standard deduction and only $4,000 of CTC instead of the applicable $24,150 deduction and $4,400 CTC."
+us,scenario_076,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model used married-filing-jointly status and a $32,200 standard deduction even though no spouse is listed and the parent qualifies as head of household. Head-of-household status requires the $24,150 deduction and corresponding brackets."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The bare assertion that deductions and exemptions yield $23,250 omits the controlling computation. Taxable income is $139,270, tentative tax is $24,211.80, and the $4,400 CTC plus $200 CDCC reduce it to $19,611.80; the submitted number reflects an unsupported shortcut."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an incorrect $23,063 standard deduction and bracket parameters, producing $24,570 rather than $24,211.80 of tentative tax. It also understated the CTC as $4,000 instead of $4,400."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-sunset regime with a reduced standard deduction, personal exemptions, reverted brackets, and a $75,000 CTC phaseout. The governing 2026 rules provide a $24,150 standard deduction and a $4,400 CTC at this $164,220 AGI."
+us,scenario_076,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied inflation-adjusted 2017 brackets, personal exemptions, and a reduced standard deduction instead of the governing 2026 rules. It also deducted the ESI premium from wages, omitted the $800 QBI deduction, and failed to apply the $4,400 CTC."
+us,scenario_076,federal_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model treated only about $1,000 of the $4,400 CTC as nonrefundable, confusing refundability limits with the credit amount that offsets existing tax liability. Because tentative tax exceeds the credit, the full $4,400 CTC plus $200 CDCC is used before refundable credits."
+us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not satisfy the output contract."
+us,scenario_076,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly derived AGI, deductions, taxable income, and credits but calculated $24,332.80 of bracket tax instead of $24,211.80. This $121 bracket-tax error carries directly into its $19,732.80 result."
+us,scenario_076,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented $1,815 of rental depreciation even though unlisted expenses are zero, reducing rental income to $2,185 and QBI to $437. Rental income remains $4,000, the QBI deduction is $800, and the standard deduction is $24,150; it also understated the CTC by $400."
+us,scenario_076,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model omitted the $800 QBI deduction and then incorrectly limited the nonrefundable CTC to $1,000 by pre-allocating $3,400 as refundable. With ample tentative tax, the full $4,400 CTC offsets tax before refundable credits, along with the $200 CDCC."
+us,scenario_076,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model applied married-filing-jointly brackets despite there being no spouse and used an internally inconsistent $16,600 MFJ standard deduction. The taxpayer files head of household, uses the $24,150 deduction and $800 QBI deduction, and receives a $4,400 rather than $4,000 CTC."
+us,scenario_076,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model included nontaxable child support, inconsistently handled the ESI premium, invented dependent deductions, and used nonsensical bracket and CTC phaseout calculations. It then submitted $18,779.30 despite explicitly computing $14,152.40, so the final value does not follow from its own reasoning."
+us,scenario_076,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the ACTC phase-in and per-child cap as creating a refundable amount without first determining how much CTC remained after offsetting federal income tax liability. The household's liability absorbs the available CTC nonrefundably, so no refundable CTC remains; it also incorrectly invoked the $400,000 married-filing-jointly phaseout threshold for a head-of-household filer."
+us,scenario_076,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model converted the $1,700-per-child ACTC ceiling directly into a $3,400 refundable credit. That figure is only a cap on otherwise eligible unused CTC, and this household's tax liability uses the CTC nonrefundably, leaving refundable CTC of zero."
+us,scenario_076,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model assumed that two qualifying children plus earned income above the ACTC phase-in threshold automatically produce $1,700 of refundable CTC per child. It omitted the required step of applying the CTC against income tax liability first; no unused CTC remains refundable."
+us,scenario_076,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, violating the required structured-output contract."
+us,scenario_076,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model correctly found that the earned-income phase-in ceiling does not bind but incorrectly treated the $1,700-per-child ACTC limit as an entitlement. The household's CTC is used against its positive federal income tax liability as a nonrefundable credit, leaving no amount for the refundable CTC calculation."
 us,scenario_076,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived $9,920 of Social Security tax and $2,320 of Medicare tax but submitted only the Social Security component. Its numeric output contradicts its own stated $12,240 total."
 us,scenario_076,payroll_tax,claude-opus-4.7,llm_error,other,False,"The model made an arithmetic error when adding its correctly computed components: $9,920 plus $2,320 equals $12,240, not $11,760."
 us,scenario_076,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model made an arithmetic error when adding its correctly computed components: $9,920 plus $2,320 equals $12,240, not $11,808."
@@ -4428,88 +4651,93 @@ us,scenario_076,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model
 us,scenario_076,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
 us,scenario_076,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated the $4,000 of rental income as earnings from self-employment and invented a 30% expense deduction that the prompt did not provide. Rental income is not subject to self-employment tax here, leaving a zero tax base and $0 liability."
 us,scenario_076,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model provided no self_employment_tax output or explanation, violating the required structured-output contract."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 household and dependent care expense deduction, leaving taxable income at $139,270 instead of $138,270. It then substituted a rough flat-rate calculation and unsupported $410 child credit for Idaho’s 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model treated the filer as single, included nontaxable child support in an intermediate income calculation, and invented a $9,380 Idaho adjustment. The filer is head of household, and the correct deductions reduce $164,220 of AGI to $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete 5.695% rate and an estimated standard deduction, omitted both the $1,000 care-expense deduction and $800 QBI deduction, and inconsistently applied a grocery credit. Idaho’s 2026 head-of-household schedule must be applied to $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an obsolete 5.695% rate, an estimated $22,500 standard deduction, and an estimated threshold instead of the 2026 parameters. It also failed to derive the $138,270 taxable-income base from all three applicable deductions."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $24,500 standard deduction while omitting the $1,000 care-expense deduction and $800 QBI deduction. It also invoked an unsupported child credit and did not apply the Idaho schedule to the correct $138,270 base."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model classified the head-of-household filer as single and deducted invented Idaho personal exemptions. It omitted the $24,150 head-of-household standard deduction, the $1,000 care-expense deduction, and the $800 QBI deduction, then used an obsolete rate and unsupported $100 credit."
-us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used an arbitrary $16,000 deduction instead of the $24,150 head-of-household standard deduction and omitted the $1,000 care-expense and $800 QBI deductions. It then inflated its own $8,443 calculation to $9,400 through unspecified adjustments."
-us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $157,740 without a valid basis, used a $23,200 standard deduction, and omitted the care-expense and QBI deductions. It also applied an incorrect 5.8% flat rate rather than the 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model asserted taxable income of $130,867 instead of deriving $138,270 from the listed deductions. It compounded that erroneous base with an incorrect 5.8% flat rate and unsupported $410 child credit."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer is a generic estimate that does not derive the $138,270 taxable-income base or apply Idaho’s 2026 head-of-household schedule. Its $8,100 result therefore reflects an unsupported shortcut rather than the required computation."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable-income figure of $129,750 instead of $138,270. It also applied an obsolete approximate 5.695% flat rate rather than Idaho’s 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an incorrect approximate 5.8% flat rate and unspecified child credits. It did not establish the correct $138,270 taxable-income base or apply the 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied only a rounded estimate with no deduction, taxable-income, rate-schedule, or credit calculation. The required derivation uses $138,270 of taxable income and yields $6,806.79 under Idaho’s 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model invoked unspecified personal and nonrefundable credits without identifying any applicable amounts or computing the tax base. It failed to apply the 2026 head-of-household schedule to $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $22,500 standard deduction and omitted the $1,000 care-expense and $800 QBI deductions, producing $141,720 instead of $138,270. It then applied an obsolete 5.695% flat rate."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave no computation supporting $6,494. The correct deduction sequence produces $138,270 of taxable income, to which Idaho’s 2026 head-of-household schedule yields $6,806.79."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model improperly relied on unspecified itemized medical assumptions even though the calculation uses the $24,150 standard deduction. It failed to combine that deduction with the $1,000 care-expense and $800 QBI deductions to obtain $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $23,350 standard deduction instead of $24,150 and omitted the $1,000 care-expense and $800 QBI deductions. It also subtracted unsupported child credits rather than applying the Idaho schedule to $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model asserted taxable income of $132,020 without deriving it from the applicable deductions and subtracted an unsupported $440 child credit. The correct taxable-income base is $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model imposed an approximate $4,930 zero-rate threshold and two unsupported $205 child credits. Idaho’s actual 2026 head-of-household schedule applied to the correctly derived $138,270 base produces $6,806.79."
-us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model stopped at $139,270 because it omitted the $1,000 household and dependent care expense deduction. It then treated 5.3% as a simple flat tax instead of applying the full 2026 head-of-household schedule to $138,270."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model replaced Idaho’s statutory calculation with an unsupported 4% effective-rate approximation. It neither derived the $138,270 taxable-income base nor applied the 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used taxable income near $141,157, explicitly denied the QBI adjustment, and omitted the $1,000 care-expense deduction. It also applied an obsolete 5.695% rate instead of the 2026 schedule to the correct $138,270 base."
-us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $157,740 starting income and a $12,585 standard deduction instead of $164,220 of AGI and the $24,150 head-of-household deduction. It omitted the $1,000 care-expense and $800 QBI deductions and applied an incorrect 5.8% rate."
-us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model used only a broad $135,000–$140,000 tax-base range and an approximate 5.3% rate. It failed to calculate the exact $138,270 base and apply Idaho’s 2026 head-of-household schedule."
-us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it failed the submission contract before any substantive tax calculation could be evaluated."
-us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 care-expense deduction and $800 QBI deduction, producing $140,070 instead of $138,270. It also imposed an incorrect $2,500 threshold and unsupported $410 child credit."
-us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $162,405, used a fictitious $3,500 head-of-household standard deduction, and subtracted unsupported $2,000 child credits. The correct calculation starts from $164,220 and reaches $138,270 after the specified deductions."
-us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used the married-filing-jointly standard deduction even though the filer is head of household, then asserted taxable income of $96,448 that does not follow from its own figures. The correct head-of-household deduction sequence yields $138,270, not $96,448."
-us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model invented a $121,620 federal taxable-income starting point, an additional $2,400 Idaho standard deduction, and a graduated 1%/3%/5.5% bracket structure. The required 2026 head-of-household computation applies Idaho’s actual schedule to $138,270."
-us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model used an outdated $120 grocery-credit amount instead of the 2026 amount of $155 per qualifying person. Applying $155 to the head and two children yields $465.
-us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that Idaho has no applicable refundable credit and omitted the refundable grocery credit. The household's income does not eliminate the $155 credit for each of its three qualifying members.
-us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model vacillated between $120 and $170 and ultimately used an incorrect $170 per-person amount. The applicable 2026 amount is $155 for each of three members, with no additional refundable credit."
-us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model misclassified Idaho's grocery credit as nonrefundable or refundable only in limited circumstances. It is included as a refundable credit here at $155 for each of the three qualifying household members.
-us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly treated the Idaho grocery credit as nonrefundable or unavailable at this income. All three household members qualify for the refundable $155 credit, totaling $465."
-us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly characterized Idaho's grocery credit as nonrefundable. The credit contributes $155 for each of the head and two children regardless of the cited high income, totaling $465."
-us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model acknowledged that the grocery credit is refundable but then omitted it on the mistaken ground that no qualifying refundable credit applied or that it was reflected elsewhere. This output must include $155 for each of three qualifying members.
-us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model substituted an invented refundable Idaho child tax credit of $205 per child for the applicable grocery credit. The entire amount is instead the $155 refundable grocery credit for all three household members, totaling $465."
-us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit and the three qualifying people but used $120 per person. The 2026 amount is $155 per person, producing $465."
-us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly imposed an income-based exclusion on Idaho refundable credits. The grocery credit remains $155 for each of the three qualifying members, totaling $465."
-us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable Idaho credit applies at this income level. It omitted the refundable grocery credit of $155 for each of three household members.
-us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $100 rather than the 2026 grocery-credit amount of $155 per qualifying person. Three qualifying members therefore generate $465, not $300."
-us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model omitted Idaho's refundable grocery credit by declaring the household ineligible. Each of the three full-year household members qualifies for $155, totaling $465."
-us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model identified the correct credit and household count but applied an outdated $120 per-person amount. The 2026 rate is $155, so three members receive $465."
-us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied a $100 grocery-credit amount instead of $155 per qualifying member for 2026. With three members, the correct computation is 3 × $155 = $465."
-us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to identify Idaho's refundable grocery credit from the household's Idaho residency and three qualifying members. That credit supplies the entire $465 output.
-us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model wrongly treated high income and the absence of special eligibility indicators as disqualifying. Idaho's grocery credit applies to all three qualifying household members at $155 each.
-us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model correctly counted three eligible household members but used $120 instead of the 2026 $155 grocery-credit amount. The correct total is $465.
-us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable Idaho credit applied. It omitted the $155 refundable grocery credit for each of the household's three full-year members.
-us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit but used an outdated $120 per-person amount. At $155 for each of three members, the credit is $465."
-us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model's zero answer omits Idaho's refundable grocery credit. The head and both children qualify for $155 each for the full year, producing $465."
-us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,The model used a $100 grocery-credit amount rather than the applicable 2026 amount of $155 per person. The head and two dependents therefore generate $465.
-us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,The model invented a full income phaseout for Idaho's grocery credit at this household's income. The credit is not phased out here and pays $155 for each of three qualifying members.
-us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly treated the grocery credit as refundable and counted three members but used $100 per person. The applicable 2026 amount is $155, totaling $465."
-us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, so the required output was missing."
-us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly identified three non-senior recipients but applied an outdated $120 grocery-credit rate. The 2026 rate is $155 per person, yielding $465."
-us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated the grocery credit as income-limited or dependent on an additional listed trigger. Idaho residency and three full-year qualifying members generate $155 each, totaling $465."
-us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model focused on earned-income-credit eligibility and overlooked Idaho's refundable grocery credit. The grocery credit applies independently and provides $155 for each of three qualifying members.
-us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model's zero answer omits Idaho's refundable grocery credit. All three household members qualify for the full-year $155 amount, totaling $465."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's reasoning correctly derived about $691 of tax from $6,906 of taxable income, but it submitted $951 instead. The submitted value does not follow its stated computation."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model repeatedly mishandled the capital losses and improperly subtracted the $8,389 employer-sponsored insurance premium from the listed gross wages. It also used a $14,600 standard deduction instead of $16,100 and then submitted $1,402 despite deriving amounts near $2 or $44."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived approximately $691 using $23,006 of AGI, a $16,100 standard deduction, and the 10% rate, but submitted $1,080. Its final value is disconnected from its stated calculation."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,350 standard deduction rather than the 2026 value of $16,100. It also invented mortgage interest from the mortgage balance and an assumed interest rate, even though unlisted expenses must be zero."
-us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly calculated roughly $691 of regular tax, then erased it by invoking qualified-business-income or other adjustments that were neither listed nor applicable. With no nonrefundable credit or further deduction, the $690.62 liability remains positive."
-us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction instead of the 2026 single-filer amount of $16,100. This overstated taxable income by $1,100 and tax by about $110."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the listed gross wages by the $8,389 employer-sponsored insurance premium and applied obsolete post-sunset concepts of an $8,300 standard deduction plus a personal exemption. The correct computation starts from the full listed wages, deducts only the allowed $3,000 capital loss above the line, and then applies the $16,100 standard deduction."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model derived $8,006 of taxable income using its deduction assumption but then asserted that unspecified credits fully offset the resulting tax. No nonrefundable credit applies, and using the correct $16,100 standard deduction leaves $6,906.18 taxed at 10%."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer-sponsored insurance premium as a reduction from the expressly listed gross wages and substituted speculative post-TCJA-sunset deductions and a personal exemption. Current 2026 parameters instead give $23,006.18 of AGI and a $16,100 standard deduction."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $8,389 insurance premium from gross wages and used an $8,300 standard deduction plus a $5,200 personal exemption. The applicable 2026 computation uses the listed wages without that premium subtraction and a $16,100 standard deduction."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly reached $23,006 of AGI but applied a standard deduction that left $9,656 of taxable income. The 2026 single standard deduction is $16,100, leaving only $6,906.18 taxable."
-us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from gross wages and invoked a personal exemption rather than applying the $16,100 standard deduction to $23,006.18 of AGI. It also submitted $67 while its explanation stated $102, so the submitted number does not follow even its own computation."
-us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,450 standard deduction instead of the 2026 single-filer amount of $16,100. That left taxable income $650 too high and overstated tax by about $65."
-us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that $26,006 of wages were below the standard deduction. After the $3,000 capital-loss deduction, AGI is $23,006.18; subtracting the $16,100 standard deduction leaves $6,906.18 of taxable income."
-us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the wage income as fully sheltered by the standard deduction without performing the taxable-income calculation. The $3,000 capital-loss deduction and $16,100 standard deduction leave $6,906.18 taxable, and no nonrefundable credit eliminates its tax."
-us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an obsolete $8,150 standard deduction and a 15% second bracket. The applicable 2026 standard deduction is $16,100, and the resulting $6,906.18 of taxable income lies wholly in the 10% bracket."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that taxable income was nonpositive. Wages less the limited $3,000 capital-loss deduction produce $23,006.18 of AGI, and the $16,100 standard deduction leaves $6,906.18 taxable."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model applied speculative post-TCJA-sunset amounts—an $8,300 standard deduction and $5,300 personal exemption—instead of the applicable $16,100 standard deduction. This overstated taxable income from $6,906.18 to $9,406."
-us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from the listed gross wages and used an $8,300 standard deduction. The correct AGI is $23,006.18 after only the allowed $3,000 capital-loss deduction, followed by the $16,100 standard deduction."
-us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable.
-us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model offset wages by the entire $7,968 capital loss instead of enforcing the $3,000 annual deduction limit, then invoked an unsupported capital-loss floor and personal-exemption credits. Only $3,000 reduces ordinary income, and no credit eliminates the tax on the remaining $6,906.18 of taxable income."
-us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly applied the $3,000 capital-loss limit but used an estimated $15,750 standard deduction instead of $16,100. This overstated taxable income and tax by $350 and $35, respectively."
-us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly asserted that nonrefundable credits erased the regular tax without identifying any applicable credit. After the $3,000 capital-loss deduction and $16,100 standard deduction, $6,906.18 remains taxable at 10%."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 household and dependent care expense deduction, leaving taxable income at $139,270 instead of $138,270. It also substituted a flat-rate calculation and invented two $205 child credits rather than applying the 2026 Idaho head-of-household schedule used for this output."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model treated the filer as single instead of head of household, included nontaxable child support in an intermediate income calculation, and invented a $9,380 Idaho adjustment. It therefore never derived the required $138,270 taxable-income base from AGI less the care-expense, standard, and QBI deductions."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an obsolete 5.695% rate and an estimated $23,625 standard deduction instead of the 2026 Idaho head-of-household parameters. It also treated the grocery credit as a nonrefundable reduction and omitted the $1,000 care-expense and $800 QBI deductions from the traced taxable-income calculation."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an obsolete 5.695% rate, an estimated $22,500 standard deduction, and an invented threshold subtraction. It failed to calculate the traced $138,270 taxable income after the $1,000 care-expense deduction, $24,150 standard deduction, and $800 QBI deduction."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model estimated both the standard deduction and tax rate and then applied an unsupported $410 child-credit adjustment. It did not use the exact $138,270 taxable-income base or the applicable 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model classified the filer as single, used a $15,000 single standard deduction, and invented three Idaho personal exemptions. It also used the obsolete 5.695% rate and a constructed $100 care credit instead of deducting the full $1,000 care expense in reaching $138,270."
+us,scenario_076,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $16,000 standard deduction and then added nearly $1,000 for unspecified “conformity nuances.” It omitted the explicit $1,000 care-expense and $800 QBI deductions and did not apply the exact 2026 head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $157,740 without a valid input basis, used a $23,200 standard deduction, and omitted the $1,000 care-expense and $800 QBI deductions. It then applied an incorrect flat 5.8% rate rather than the 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used an unexplained taxable income of $130,867 rather than the traced $138,270. It also applied a 5.8% flat rate and subtracted an unsupported $410 state child credit."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The answer is only a generic estimate from AGI after unspecified deductions. It does not perform the required derivation to $138,270 or apply the 2026 Idaho head-of-household rate schedule that yields $6,806.79."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an unexplained $129,750 taxable-income figure and the obsolete 5.695% rate. The required deductions produce taxable income of $138,270, which must be subjected to the 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an approximate 5.8% flat rate and unspecified state child credits. That shortcut replaces the exact $138,270 taxable-income derivation and the applicable 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model supplied no deduction, credit, or rate calculation supporting $6,985. The required computation uses taxable income of $138,270 and the 2026 Idaho head-of-household schedule, which produces $6,806.79."
+us,scenario_076,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model relied on unspecified personal and nonrefundable credits without identifying any applicable computation. It failed to show the traced deductions yielding $138,270 or the 2026 head-of-household schedule calculation."
+us,scenario_076,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $22,500 standard deduction and omitted both the $1,000 care-expense deduction and $800 QBI deduction. It consequently taxed $141,720 at an obsolete 5.695% flat rate instead of applying the 2026 schedule to $138,270."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model gave a bare estimate without identifying the applicable deductions or schedule. The correct computation derives $138,270 of taxable income and applies the 2026 Idaho head-of-household rate schedule to reach $6,806.79."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model introduced unspecified itemized medical assumptions even though the traced calculation uses the $24,150 standard deduction. It failed to apply the separate $1,000 care-expense and $800 QBI deductions and did not calculate tax from the resulting $138,270 base."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $23,350 standard deduction, omitted the $1,000 care-expense and $800 QBI deductions, and subtracted two invented $173 child credits. Its flat 5.3% calculation therefore does not implement the applicable head-of-household schedule on $138,270."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $132,020 taxable-income base and an unsupported $440 child credit. The listed deductions instead produce $138,270, to which the 2026 Idaho head-of-household schedule applies."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model imposed an approximate $4,930 zero-rate threshold and subtracted two unsupported $205 child credits. It did not use the actual 2026 Idaho head-of-household schedule on the traced $138,270 taxable-income base."
+us,scenario_076,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 household and dependent care expense deduction, producing $139,270 rather than $138,270. It then treated 5.3% as a simple flat tax instead of applying the complete 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model replaced the statutory computation with an approximate 4% effective rate on wages. It did not derive taxable income of $138,270 or apply the 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used the obsolete 5.695% rate and asserted no QBI conformity, thereby omitting the traced $800 QBI deduction. It also failed to include the $1,000 care-expense deduction and did not reach the required $138,270 taxable income."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $12,450 standard deduction instead of the $24,150 head-of-household amount and omitted the $1,000 care-expense and $800 QBI deductions. It also applied the obsolete 5.695% rate to the resulting overstated base."
+us,scenario_076,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used a $12,585 standard deduction, omitted the $1,000 care-expense and $800 QBI deductions, and taxed the overstated base at 5.8%. The required taxable income is $138,270 under the head-of-household deductions."
+us,scenario_076,state_income_tax_before_refundable_credits,inkling,llm_error,other,False,"The model used only a broad $135,000–$140,000 range and an approximate 5.3% rate. It did not calculate the exact $138,270 taxable-income base or apply the full 2026 Idaho head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required numeric answer was missing."
+us,scenario_076,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 care-expense deduction and $800 QBI deduction, producing $140,070 rather than $138,270. It also imposed a $2,500 threshold and subtracted two unsupported $205 child credits instead of applying the actual 2026 head-of-household schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $162,405, used an invented $3,500 head-of-household standard deduction, and subtracted two nonexistent $1,000 state child credits. Those errors bypass the traced $138,270 taxable-income derivation and the applicable 2026 schedule."
+us,scenario_076,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model invented two $1,500 dependent deductions and omitted the $1,000 care-expense and $800 QBI deductions. It then applied 5.305% as a simple flat rate rather than using the 2026 Idaho head-of-household schedule on $138,270."
+us,scenario_076,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model used a married-filing-jointly standard deduction even though the filer is head of household, and its unexplained $96,448 taxable income does not follow from its own arithmetic. It also omitted the care-expense and QBI deductions and applied an incorrect 5.8% rate."
+us,scenario_076,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model began from an unexplained $121,620 federal taxable-income figure, deducted an invented $2,400 Idaho standard deduction, and applied an obsolete multi-bracket schedule. The required calculation instead yields $138,270 after the specified deductions and applies Idaho’s 2026 head-of-household schedule."
+us,scenario_076,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,The model used an outdated $120 grocery-credit amount instead of the 2026 amount of $155 per qualifying person. Applying $155 to all three household members yields $465.
+us,scenario_076,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly denied the existence of an applicable refundable Idaho credit. Idaho's grocery credit is refundable, and all three household members qualify for $155, regardless of this household's high income."
+us,scenario_076,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model alternated between unsupported $120 and $170 per-person amounts and ultimately used $170. The applicable 2026 grocery-credit amount is $155 for each of the three members, with no additional refundable credit."
+us,scenario_076,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model misclassified Idaho's grocery credit as nonrefundable or refundable only under limited circumstances. It is a refundable $155-per-person credit here, producing $465 for three qualifying members."
+us,scenario_076,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly treated the Idaho grocery credit as nonrefundable or unavailable at this income. Each of the three members qualifies for the refundable $155 credit, totaling $465."
+us,scenario_076,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified Idaho's grocery credit as nonrefundable. The refundable grocery credit supplies $155 for each of the three household members, totaling $465."
+us,scenario_076,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model acknowledged that the grocery credit is refundable but then excluded it based on income and asserted it was reflected elsewhere. The credit belongs in state_refundable_credits and provides $155 for each of three members, totaling $465."
+us,scenario_076,state_refundable_credits,deepseek-v4-pro,llm_error,other,False,"The model substituted a purported refundable Idaho child tax credit of $205 per child for the applicable grocery credit. The entire amount is instead the $155 grocery credit for the head and both children, totaling $465."
+us,scenario_076,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly identified the refundable grocery credit and all three recipients but used $120 per person. The 2026 amount is $155 per person, so the total is $465."
+us,scenario_076,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly imposed an income-based exclusion on Idaho's refundable grocery credit. All three household members qualify for $155 each, yielding $465."
+us,scenario_076,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable Idaho credit applies at this income level. Idaho's grocery credit provides $155 to each of the three qualifying members, totaling $465."
+us,scenario_076,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used $100 rather than the 2026 grocery-credit amount of $155 per qualifying person. Three qualifying members therefore produce $465, not $300."
+us,scenario_076,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model omitted the refundable Idaho grocery credit. The head and two children each qualify for the full-year $155 amount, totaling $465."
+us,scenario_076,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly counted three qualifying people but applied an outdated $120 per-person grocery credit. At the 2026 amount of $155 each, the credit totals $465."
+us,scenario_076,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,The model used a $100 per-member grocery-credit parameter instead of the 2026 amount of $155. Multiplying the correct amount by three members gives $465.
+us,scenario_076,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to apply Idaho's refundable grocery credit despite the household's three qualifying members. Each receives $155 for the full year, producing $465."
+us,scenario_076,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated high income as disqualifying the household from Idaho's grocery credit. That income restriction does not eliminate the credit here, and three members at $155 each yield $465."
+us,scenario_076,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model identified the correct credit and recipient count but used $120 instead of the 2026 $155 per-person amount. The correct multiplication is 3 × $155 = $465.
+us,scenario_076,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model omitted the applicable refundable Idaho grocery credit. The head and two children qualify for $155 each, totaling $465."
+us,scenario_076,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied an outdated $120 grocery-credit amount to the correct three-person count. The 2026 amount is $155 per person, producing $465."
+us,scenario_076,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to recognize Idaho's refundable grocery credit as applicable. All three household members qualify for $155 each, so refundable state credits equal $465."
+us,scenario_076,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly counted the head and two dependents but used $100 per person. Idaho's 2026 grocery-credit amount is $155 per qualifying member, totaling $465."
+us,scenario_076,state_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,The model used a $100 permanent-law parameter rather than the applicable 2026 grocery-credit amount of $155. Three full-year qualifying members generate $465.
+us,scenario_076,state_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model incorrectly phased out Idaho's grocery credit at this household's income. No such phaseout removes the credit here, and the three members receive $155 each for a total of $465."
+us,scenario_076,state_refundable_credits,inkling,llm_error,thresholds_rates,False,The model used $100 per household member rather than the 2026 grocery-credit amount of $155. Applying the correct amount to three members yields $465.
+us,scenario_076,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required output contract."
+us,scenario_076,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model correctly included the head and both children but used $120 per person. The applicable 2026 amount is $155 each, so the total is $465."
+us,scenario_076,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly treated the grocery credit as income-limited or dependent on an additional listed trigger. Each of the three household members qualifies for the refundable $155 credit, totaling $465."
+us,scenario_076,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,The model correctly counted three non-senior household members but used $100 instead of $155 per person. The absence of an aged supplement does not change the base calculation of 3 × $155 = $465.
+us,scenario_076,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model focused on earned-income-credit eligibility and overlooked Idaho's refundable grocery credit. That credit applies to all three household members at $155 each, totaling $465."
+us,scenario_076,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Idaho's applicable refundable grocery credit. The three full-year qualifying household members each receive $155, producing $465."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $691, including the $3,000 capital-loss limit, $16,100 standard deduction, and 10% rate, but submitted $951 instead. Its submitted value is disconnected from its own completed calculation."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model repeatedly miscomputed income by subtracting the entire capital loss and then separately subtracting the listed employer-sponsored insurance premiums from wages. Gross wages remain $26,006.18, only $3,000 of net capital loss offsets ordinary income, and the $16,100 standard deduction leaves $6,906.18 taxable; its final $1,402 is also unsupported by any calculation in its explanation."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income of about $6,906 and tax of about $691, then submitted $1,080 without applying any stated adjustment that produces that number. It abandoned its own correct computation at the final-value step."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an estimated 2026 single standard deduction of $15,350 instead of the applicable $16,100 amount. The correct deduction reduces $23,006.18 of AGI to $6,906.18 of taxable income, yielding $690.62 at 10%."
+us,scenario_077,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model correctly derived roughly $691 of regular income tax but then invented an unspecified qualified-business-income or other adjustment to reduce it to zero. No such deduction or nonrefundable credit applies to the listed facts, so the $690.62 liability remains."
+us,scenario_077,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $15,000 standard deduction instead of the applicable 2026 single deduction of $16,100. This overstated taxable income by $1,100 and tax by approximately $110."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from the stated gross wages and replaced the applicable $16,100 standard deduction with a projected standard-deduction-plus-personal-exemption scheme. Starting from the stated wages and allowing only the $3,000 capital-loss deduction produces $23,006.18 of AGI and $6,906.18 of taxable income."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model calculated positive taxable income but asserted that unspecified available credits fully offset the tax. The household has no listed facts supporting a nonrefundable credit, so the regular tax on $6,906.18 remains $690.62."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced gross wages by the $8,389 employer-sponsored insurance premiums and applied a projected post-TCJA standard-deduction-plus-personal-exemption regime. The correct calculation retains the stated wages, deducts $3,000 of capital loss, and applies the $16,100 standard deduction."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored insurance premiums from the stated gross wages and used an inapplicable personal exemption with a reduced standard deduction. The applicable calculation uses $23,006.18 of AGI and a $16,100 standard deduction, leaving $6,906.18 taxable."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly reached $23,006 of AGI but implicitly used a deduction far below the applicable $16,100 standard deduction. Applying $16,100 leaves $6,906.18 taxable and produces $690.62, not $965.60."
+us,scenario_077,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted $8,389 of employer-sponsored insurance premiums from gross wages and invoked an inapplicable personal exemption. It also submitted $67 while its explanation concluded with $102, so neither its stated computation nor its submitted value reflects the $690.62 tax on $6,906.18 of taxable income."
+us,scenario_077,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,450 standard deduction rather than the applicable $16,100 deduction. That error overstated taxable income by $650 and tax by approximately $65."
+us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that $26,006 of wages is below the standard deduction. After the permitted $3,000 capital-loss deduction, AGI is $23,006.18, which exceeds the $16,100 standard deduction by $6,906.18."
+us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the filer's income as producing no tax and referred generically to nonrefundable credits despite no qualifying credit facts. The standard deduction leaves $6,906.18 taxable, and no nonrefundable credit offsets its $690.62 tax."
+us,scenario_077,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an $8,150 standard deduction instead of $16,100 and then applied an inapplicable 15% second bracket. The full $6,906.18 of correctly computed taxable income lies in the 10% bracket."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the capital-loss deduction and standard deduction reduce taxable income to zero. Only $3,000 of the net capital loss offsets wages, leaving $23,006.18 of AGI and $6,906.18 after the $16,100 standard deduction."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a projected TCJA-sunset regime with an $8,300 standard deduction and $5,300 personal exemption instead of the applicable $16,100 standard deduction. This left $2,500 too much taxable income and overstated tax accordingly."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model substituted an $8,300 standard deduction plus a $5,300 personal exemption for the applicable $16,100 standard deduction. The correct deduction leaves $6,906.18 taxable rather than $9,406."
+us,scenario_077,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the listed $8,389 employer-sponsored insurance premiums from gross wages and used only an $8,300 standard deduction. The stated wages remain in income, and applying the $3,000 capital-loss deduction plus the $16,100 standard deduction yields $6,906.18 taxable."
+us,scenario_077,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output for the requested variable. It therefore failed the required structured-output contract without performing a usable tax computation.
+us,scenario_077,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model initially offset wages by the entire $7,968 capital loss instead of enforcing the $3,000 annual deduction limit, then invented a $400 adjustment and personal-exemption credits to reach zero. Only $3,000 reduces ordinary income, and no listed credit eliminates the $690.62 liability."
+us,scenario_077,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model correctly applied the $3,000 capital-loss limit but used a $15,750 standard deduction instead of $16,100. The additional $350 deduction reduces taxable income to $6,906.18 and tax to $690.62."
+us,scenario_077,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted that unspecified nonrefundable credits eliminate the remaining regular tax. No qualifying nonrefundable credit is supported by the household facts, so tax on the $6,906.18 remaining after deductions is $690.62."
 us,scenario_077,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly determined that the greater-of-earned-income-or-AGI rule uses $26,006 and fully phases out the childless EITC, then submitted $56 despite its own zero-credit computation. No other refundable credit supplies the submitted amount."
 us,scenario_077,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model incorrectly left $9 of childless EITC after acknowledging $26,006 of wages. EITC phaseout uses the greater of earned income or AGI, so the capital-loss deduction does not prevent the credit from being fully phased out."
 us,scenario_077,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model reduced $26,006 of wages by the $8,389 employer-sponsored insurance premium and treated $17,617 as EITC earned income. Employer premiums do not reduce EITC earned income here, so the phaseout must use $26,006 and eliminates the childless credit."
@@ -4544,75 +4772,80 @@ us,scenario_077,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The m
 us,scenario_077,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly assumed that the $8,389 employer-sponsored insurance premium was pre-tax for FICA and reduced payroll-taxable wages to $17,617. Using the full $26,006 wage base yields $1,989.47 rather than the rounded $1,348.00."
 us,scenario_077,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required output contract rather than completing the employee Social Security and Medicare calculation."
 us,scenario_077,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_077,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly treated Louisiana broad-based categorical eligibility as waiving the SNAP net-income test and treated a mortgage balance and non-elderly medical spending as deductible expenses. Net income after the permitted earned-income and standard deductions exceeds the one-person limit, and its $4,218 estimate also exceeds its own stated $298 monthly maximum allotment of $3,576 annually."
-us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used negative capital gains to reduce SNAP countable wage income, even though investment capital losses do not offset earned income in the SNAP calculation. With only the 20% earned-income deduction and standard deduction, countable net income remains above the eligibility limit."
-us,scenario_077,snap,gemini-3.7-flash,llm_error,other,False,"The model's explanation correctly states that income exceeds the one-person SNAP net-income limit and explicitly ends with “value = 0,” but it submitted 276 as the numeric output. The failure is an internal answer-contract mismatch rather than a SNAP-rule error."
-us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income and declared it eligible without applying the one-person SNAP net-income test. After the permitted 20% earned-income and standard deductions, countable monthly income exceeds the limit, and the listed mortgage balance, capital losses, and medical expenses do not create additional allowable deductions for this non-elderly, nondisabled household."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used a $12,500 deduction instead of Louisiana's $12,835 standard deduction, then abandoned its own resulting $315.18 computation and asserted $496 without any valid adjustment. The correct taxable income is $10,171.18 and 3% produces $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely treated Louisiana as having no individual income tax. Louisiana imposes a 3% tax in 2026, and this filer has $10,171.18 of Louisiana taxable income, producing $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $12,500 single-filer deduction instead of $12,835. Subtracting the correct deduction from $23,006.18 leaves $10,171.18, whose 3% tax is $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model substituted a $12,500 combined exemption for Louisiana's $12,835 standard deduction. That understated the deduction by $335 and overstated the 3% liability by about $10.05."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly sketched a liability near $315 using its assumed deduction, then replaced it with $446 based on an invented PolicyEngine rounding and capital-loss adjustment. PolicyEngine's AGI is $23,006.18, and ordinary rounding cannot transform the $305.14 statutory computation into $446."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied an obsolete $4,500 personal exemption and a federal-income-tax deduction instead of Louisiana's 2026 $12,835 standard deduction. The 2026 calculation taxes $10,171.18 at the flat 3% rate, yielding $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first computed roughly $315 from a $12,500 deduction, then asserted $448 through unspecified federal adjustments and personal-exemption credits that do not produce that amount. Using the $12,835 deduction directly yields $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $12,500 standard deduction instead of $12,835. The additional $335 deduction reduces taxable income to $10,171.18 and the 3% tax to $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $8,389 employer-sponsored insurance premium from the explicitly stated gross wages, reducing AGI to $14,617, and then applied the obsolete 1.85% bracket and $4,500 exemption. The applicable AGI is $23,006.18, followed by the $12,835 deduction and 3% flat rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unexplained $158 does not follow the traced calculation. AGI of $23,006.18 less the $12,835 standard deduction leaves $10,171.18, and the 3% rate produces $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly removed the employer-sponsored insurance premium from stated gross wages to obtain $14,617 of AGI, then double-counted obsolete $4,500 deductions and used a 1.85% rate. The correct sequence uses $23,006.18 of AGI, one $12,835 standard deduction, and the 3% flat rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the listed employer premium as an additional subtraction from gross wages and therefore understated AGI as $14,617. It also used the former $4,500 exemption and 1.85% bracket rather than the 2026 $12,835 deduction and 3% flat rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $315.18 reflects subtraction of only $12,500 from approximately $23,006. Louisiana's applicable standard deduction is $12,835, leaving $10,171.18 taxable and $305.14 of tax."
-us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model submitted $152 while its explanation declared a final value of $130, violating the required same-value contract. Neither figure follows the traced $23,006.18 AGI minus $12,835 deduction and 3% rate calculation."
-us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used a $12,500 deduction rather than the applicable $12,835 amount. This overstated taxable income by $335 and tax by $10.05."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model incorrectly treated the deduction as an income threshold that eliminated all liability. After the $12,835 deduction, the filer still has $10,171.18 of taxable income subject to the 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model assumed low wages and unspecified credits reduced the liability to zero without computing Louisiana taxable income. The $12,835 standard deduction leaves $10,171.18 taxable, and no identified nonrefundable credit eliminates the resulting $305.14 tax."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model substituted a $12,500 standard deduction for Louisiana's $12,835 deduction. Applying 3% to the correct $10,171.18 taxable income gives $305.14 rather than $315.18."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model estimated the single deduction at $12,500 rather than applying the $12,835 parameter. The correct deduction lowers taxable income by another $335 and tax by $10.05."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the wrong Louisiana standard-deduction parameter, $12,500 instead of $12,835. With the correct deduction, taxable income is $10,171.18 and tax is $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model deducted both a $12,500 standard deduction and a separate $4,500 personal exemption. Louisiana's traced 2026 computation instead subtracts the single $12,835 standard deduction, leaving $10,171.18 taxable."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and exemptions exhausted the filer’s income. AGI of $23,006.18 exceeds the $12,835 standard deduction by $10,171.18, so the 3% tax remains positive."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model applied Louisiana's obsolete $4,500 exemption and graduated 1.85% and 3.5% rates. For 2026, the relevant computation uses the $12,835 standard deduction and flat 3% rate, producing $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly reduced stated gross wages by the employer-sponsored insurance premium to reach $14,617 of AGI, then used an obsolete $4,500 deduction and 1.85% rate. The applicable figures are $23,006.18 of AGI, a $12,835 deduction, and a 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used a $12,500 standard deduction and rounded the resulting tax to $315. Louisiana's $12,835 deduction leaves $10,171.18 taxable, yielding $305.14 at 3%."
-us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable. It therefore never performed the required $23,006.18 minus $12,835 calculation or submitted the resulting $305.14."
-us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used a $12,500 standard deduction instead of $12,835. That $335 parameter error accounts for the $10.04 difference after cent rounding."
-us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Louisiana has no state income tax. Louisiana's 2026 flat 3% tax applies to this filer's $10,171.18 of taxable income."
-us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model deducted the full $7,968 capital loss instead of applying the $3,000 federal limitation embedded in AGI, then used an obsolete $4,500 exemption and graduated rates. The correct AGI is $23,006.18, from which Louisiana subtracts $12,835 before applying its flat 3% rate."
-us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The unexplained $347.24 is inconsistent with the stated capital-loss limitation and Louisiana deduction computation. The allowed loss produces $23,006.18 of AGI; subtracting $12,835 and applying 3% yields $305.14."
+us,scenario_077,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated broad-based categorical eligibility as waiving the net-income requirement and invented an excess-shelter deduction from the mortgage balance, even though no mortgage payment, rent, or utility expense was listed. It also used medical deductions unavailable to this 37-year-old nondisabled household and returned $4,218, which exceeds its own stated one-person maximum of about $298 per month."
+us,scenario_077,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced SNAP countable earned income by the household’s capital losses, conflating tax AGI treatment with SNAP income rules. The wages remain countable subject only to applicable SNAP deductions, so the capital losses do not create the partial $1,656 allotment."
+us,scenario_077,snap,gemini-3.7-flash,llm_error,other,False,"The model’s explanation correctly states that the one-person household exceeds the SNAP net-income limit and explicitly concludes “value = 0,” but it submitted 276. The numeric output contradicts its own eligibility conclusion and should have been 0."
+us,scenario_077,snap,grok-4.3,llm_error,thresholds_rates,False,"The model labeled the household low-income without applying the one-person SNAP net-income and benefit calculations to its $26,006 of wages. After the allowable earned-income and standard deductions, the household does not generate an eligible allotment, so the asserted $2,400 benefit has no valid computation."
+us,scenario_077,snap,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly awarded the $23 minimum allotment after determining that the regular formula produced zero. Broad-based categorical eligibility does not entitle a household failing the applicable net-income or benefit test to the minimum allotment; the minimum applies only to an otherwise eligible one- or two-person household.
+us,scenario_077,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 Louisiana standard deduction instead of $12,835, then abandoned its own resulting $315.18 calculation and asserted $496 without any valid adjustment. The traced taxable income is $10,171.18, whose 3% tax is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model falsely treated Louisiana as a state without an individual income tax. Louisiana imposes a 3% tax here on $10,171.18 of taxable income, producing $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model applied a $12,500 single deduction rather than Louisiana's applicable $12,835 deduction. That understated the deduction by $335 and produced about $10.05 too much tax."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used $12,500 as the combined Louisiana deduction/exemption instead of subtracting the applicable $12,835 standard deduction. Correct taxable income is $10,171.18 rather than $10,506."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model first used the wrong $12,500 deduction to derive about $315, then replaced that result with $446 under an invented PolicyEngine rounding and capital-loss adjustment. Neither rounding nor capital-loss treatment can turn its stated taxable-income computation into $446; the traced calculation yields $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $4,500 personal exemption and a federal-income-tax deduction instead of Louisiana's 2026 $12,835 standard deduction. This left $17,705 taxable rather than $10,171.18 and consequently overstated the tax."
+us,scenario_077,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model derived roughly $315 using the wrong $12,500 deduction and then asserted that unspecified federal-based adjustments increased the result to $448. Louisiana's traced calculation subtracts $12,835 from $23,006.18 and applies 3%, with no such upward adjustment."
+us,scenario_077,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction instead of $12,835. The extra $335 of deductible income reduces taxable income to $10,171.18 and tax to $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the $8,389 employer-sponsored insurance premium from the separately stated gross wages to manufacture a $14,617 AGI, then used an obsolete $4,500 deduction and 1.85% bracket. The computation instead uses $23,006.18 AGI, a $12,835 deduction, and the 3% rate."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model supplied no computation supporting $158. The traced derivation subtracts $12,835 from $23,006.18 and applies 3%, producing $305.14, so its number omits or misstates a material part of that calculation."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by the employer insurance premium to obtain $14,617 of AGI, double-counted separate $4,500 deductions, and applied the obsolete 1.85% rate. Louisiana instead uses $23,006.18 AGI less one $12,835 standard deduction at 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated employer-sponsored insurance premiums as a subtraction from the supplied gross wages, reducing AGI to $14,617, and then applied an obsolete $4,500 deduction and 1.85% rate. The correct state base is $23,006.18 less $12,835, taxed at 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its $315.18 result implies that it subtracted only $12,500 from approximately $23,006 before applying 3%. The applicable deduction is $12,835, leaving $10,171.18 taxable and $305.14 of tax."
+us,scenario_077,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model submitted $152 while its explanation asserted a different final amount of $130, and neither amount follows the Louisiana computation. The required AGI-minus-$12,835 base taxed at 3% yields $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 Louisiana standard deduction rather than $12,835. This overstated taxable income by $335 and tax by $10.05."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model invented a tax-free lowest-bracket threshold after deductions. Louisiana has $10,171.18 of positive taxable income here, all subject to the 3% rate, so liability is $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model set the liability to zero merely because wages were low and no credits were specified, without calculating Louisiana taxable income. AGI of $23,006.18 less the $12,835 deduction leaves $10,171.18 taxed at 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 single standard deduction instead of $12,835. Correcting that parameter changes taxable income from about $10,506 to $10,171.18 and tax from $315.18 to $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model estimated the Louisiana single deduction at $12,500 rather than using $12,835. The correct deduction produces $10,171.18 of taxable income and $305.14 of tax."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model subtracted $12,500 rather than the applicable $12,835 Louisiana standard deduction. Its taxable base was therefore $335 too high."
+us,scenario_077,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used the wrong $12,500 standard deduction and then deducted an additional $4,500 personal exemption that does not belong in this 2026 calculation. Louisiana subtracts a single $12,835 standard deduction, leaving $10,171.18 taxable."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly declared taxable income zero or negative after deductions. Subtracting the $12,835 Louisiana deduction from $23,006.18 leaves positive taxable income of $10,171.18."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used an obsolete $4,500 exemption and obsolete graduated rates of 1.85% and 3.5%. The 2026 calculation uses a $12,835 standard deduction and a flat 3% rate."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 standard deduction instead of $12,835. This made its taxable income $10,506 rather than $10,171.18 and its tax about $10 too high."
+us,scenario_077,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly reduced gross wages by employer-sponsored insurance premiums to reach $14,617 of AGI, then applied an obsolete $4,500 deduction and 1.85% rate. The traced inputs produce $23,006.18 AGI, a $12,835 deduction, and tax at 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used a $12,500 Louisiana standard deduction rather than $12,835. The correct taxable base is $10,171.18, not approximately $10,506."
+us,scenario_077,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. It therefore failed the required structured-output contract.
+us,scenario_077,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted a $12,500 standard deduction instead of the applicable $12,835 amount. Its taxable income and resulting 3% tax were consequently overstated."
+us,scenario_077,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely asserted that Louisiana has no state income tax. Louisiana taxes the $10,171.18 state taxable income at 3%, producing $305.14."
+us,scenario_077,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted both the federal standard deduction and a $4,500 personal exemption from Louisiana AGI. The state calculation instead subtracts Louisiana's $12,835 standard deduction, leaving $10,171.18 taxable."
+us,scenario_077,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model deducted the entire $7,968 net capital loss instead of applying the $3,000 AGI limitation, used a $4,500 exemption, and applied obsolete graduated rates. The correct path uses $23,006.18 AGI, subtracts $12,835, and taxes the resulting $10,171.18 at a flat 3%."
+us,scenario_077,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model gave only a generic claim about a standard deduction and capital-loss limitation, with no figures that can produce $347.24. Applying those items correctly gives $23,006.18 of AGI, $10,171.18 of taxable income, and $305.14 of tax."
 us,scenario_077,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former 5% refundable EITC in tax year 2026. That state credit is unavailable for 2026, so multiplying an estimated federal EITC by 5% incorrectly produced $10.53 instead of zero."
 us,scenario_077,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model applied Louisiana’s former refundable EITC rate of 5% to a purported $108 federal EITC. Louisiana provides no refundable EITC for tax year 2026, so the state refundable credit is zero."
 us,scenario_077,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model incorrectly carried Louisiana’s former 5% refundable EITC into 2026 and also assigned a $625 federal EITC to a childless filer with $26,006 of earnings, above the childless-credit phaseout range. The unavailable state credit cannot produce the claimed $31.25."
 us,scenario_077,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required structured-output contract."
 us,scenario_077,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $1,632 state/local tax refund to current-year AGI and counted only $26,460 of SALT instead of the $36,233.77 deduction. It therefore used $145,467 of taxable income rather than $134,061.22, and its displayed bracket components also sum to $27,510.08, not $33,499."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly included the $1,632 refund in AGI, deducted the suspended $2,606 unreimbursed employee expense above the line, and omitted $9,773.77 of deductible SALT. Those errors produced $142,861 of taxable income instead of $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $10,000 SALT cap rather than the applicable 2026 SALT deduction of $36,233.77 and also added the $1,632 refund to AGI. It consequently taxed $161,927 instead of $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap and included the $1,632 refund in AGI, overstating taxable income by $27,865.78. It then abandoned its own $31,424 bracket calculation and submitted the unsupported $36,757."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model limited SALT to $10,000 instead of $36,233.77 and treated the $1,632 refund as current taxable income. This yielded $161,927 of taxable income rather than $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used the obsolete $10,000 SALT cap and added the $1,632 refund to AGI. Its taxable-income base was therefore $161,927 rather than $134,061.22, regardless of its subsequent approximate bracket indexing."
-us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 instead of deducting $36,233.77, so its stated taxable income was overstated. It then added payroll-tax concepts and unexplained rounding to income tax, even though its own regular-income-tax arithmetic produced about $31,069 rather than $42,800."
-us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied pre-TCJA rates and a personal exemption even though the applicable 2026 computation uses the 10%/12%/22%/24% schedule and no personal exemption. It also added the refund to AGI and approximated the SALT deduction rather than using the traced $36,233.77."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, restored a $5,150 personal exemption, and applied the old 10%/15%/25%/28% brackets. The applicable computation instead deducts $64,443.77 from $198,505 and applies the 2026 10%/12%/22%/24% brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction and ignored the larger $64,443.77 itemized deduction consisting of mortgage interest and SALT. Taxable income is $134,061.22, not $183,505."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly restored personal exemptions and asserted that AMT raised the liability under a post-TCJA-sunset regime. The regular 2026 computation on $134,061.22 of taxable income yields the requested $24,772.69 without that invented adjustment."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed a TCJA sunset and subtracted a personal exemption that is absent from the applicable computation. It also approximated state income tax rather than using the traced total SALT deduction of $36,233.77, so it did not reach taxable income of $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model added the $1,632 refund to AGI and improperly subtracted a $5,050 personal exemption under its sunset assumption. The applicable derivation has AGI of $198,505, itemized deductions of $64,443.77, and no personal exemption."
-us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount implies that the model failed to use the full traced $64,443.77 itemized deduction and the resulting $134,061.22 taxable income. Applying the 2026 brackets to that base yields $24,772.69, not $31,316.32."
-us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model ignored the larger itemized deduction and then invented a $12,000 child/dependent-care credit despite the household containing no child, dependent, qualifying care expense, or care provider. A single filer cannot claim CDCC for the filer’s own ordinary expenses."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap instead of using the $36,233.77 SALT deduction. Its $42,034 also does not follow from either the standard deduction or the itemized deduction it described under the applicable 2026 brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model’s approximate amount does not reflect the traced $36,233.77 SALT deduction combined with $28,210 of mortgage interest. Those deductions reduce taxable income to $134,061.22 and produce $24,772.69 before refundable credits."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model wrongly added the $1,632 refund to AGI and counted only the $26,460 property tax as SALT, omitting $9,773.77 of the allowable SALT deduction. It therefore used $145,467 rather than $134,061.22 of taxable income."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The amount implies that the model did not use the full $36,233.77 SALT deduction together with the $28,210 mortgage-interest deduction. The correct itemized total is $64,443.77, leaving taxable income of $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $1,632 refund as taxable and limited SALT to the $26,460 real-estate tax. This overstated taxable income at $145,467 instead of $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model counted only mortgage interest and real-estate tax, omitting $9,773.77 of the traced SALT deduction. The full $64,443.77 itemized deduction produces $134,061.22 of taxable income and $24,772.69 of tax."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $41,500 answer is inconsistent with taxing the traced $134,061.22 of taxable income under the 2026 single-filer brackets. It reflects failure to apply the full $64,443.77 itemized deduction or the progressive bracket calculation."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model wrongly added the refund to AGI, introduced a personal exemption, and applied pre-TCJA-sunset brackets. The applicable calculation has no personal exemption and applies the 2026 10%/12%/22%/24% brackets to $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and used pre-TCJA brackets. It also added the refund to AGI and approximated SALT instead of using AGI of $198,505 and the traced $36,233.77 SALT deduction."
-us,scenario_078,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added the $1,632 refund to AGI and omitted $9,773.77 of deductible SALT by counting only real-estate tax. It therefore taxed $145,467 rather than $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output for the requested variable. This is a missing-output contract failure rather than a substantive tax calculation.
-us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $1,632 refund in AGI and treated $26,460 as the entire SALT deduction despite the traced $36,233.77 amount. Its taxable income was consequently $145,467 instead of $134,061.22."
-us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite $198,505 of wage income and provided no computation. After the $64,443.77 itemized deduction, $134,061.22 remains taxable and generates $24,772.69 under the 2026 brackets."
-us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model imposed an obsolete $10,000 SALT cap and added the $1,632 refund to AGI, producing $161,927 of taxable income. It also selected 2024 bracket thresholds after discussing 2026, but the decisive base is $134,061.22 under the applicable 2026 rules."
-us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model constructed an unsupported AMT calculation, including a fabricated offset and an extra $1,127.51 rounding adjustment, instead of applying regular tax to the traced taxable income. It also used only $54,670 of itemized deductions and added the refund to AGI, missing the $64,443.77 deduction and $134,061.22 taxable-income result."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included the $1,632 refund in AGI and limited SALT to the listed $26,460 property tax, omitting deductible Maryland income tax that raises SALT to $36,233.77. It also made an arithmetic error: its displayed bracket components total $27,510.08, not $33,510 or $33,499."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $2,606 of unreimbursed employee expenses, included the $1,632 refund in AGI, and omitted deductible Maryland income tax from SALT. The correct inputs are $198,505 AGI and $64,443.77 of itemized deductions, producing $134,061.22 of taxable income."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model imposed an obsolete $10,000 SALT cap and included the $1,632 refund in AGI. PolicyEngine allows $36,233.77 of SALT in 2026 and uses $198,505 AGI, reducing taxable income to $134,061.22 rather than $161,927."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap and included the refund in AGI, instead of using $36,233.77 of SALT and $198,505 AGI. Its final $36,757 also contradicts its own bracket calculation of about $31,424."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 and treated the $1,632 refund as current-year AGI. Those errors inflated taxable income from $134,061.22 to $161,927."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model applied an obsolete $10,000 SALT cap, omitted deductible Maryland income tax, and included the refund in AGI. The allowable SALT deduction is $36,233.77 and AGI is $198,505."
+us,scenario_078,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 rather than allowing $36,233.77, leaving taxable income overstated. It then abandoned its own regular-income-tax calculation of about $31,069 and added payroll-tax concepts that do not belong in federal income tax before refundable credits."
+us,scenario_078,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA personal exemption and restored 10/15/25/28 percent brackets instead of the operative 2026 brackets. It also included the $1,632 refund in AGI; the correct computation has no personal exemption and taxes $134,061.22 under the 2026 bracket schedule."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed TCJA expiration, subtracted a personal exemption, and used restored 10/15/25/28 percent brackets. The operative computation instead uses $64,443.77 of itemized deductions, no personal exemption, and the 2026 single-filer brackets on $134,061.22."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used only a standard deduction and ignored the much larger $64,443.77 itemized deduction comprising mortgage interest and SALT. Itemization reduces taxable income to $134,061.22, not $183,505."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly restored personal exemptions and post-sunset rules and then invoked AMT. The regular 2026 computation uses no personal exemption and directly taxes $134,061.22 after $64,443.77 of itemized deductions."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model assumed TCJA sunset and subtracted a personal exemption that is absent from the reference computation. It also estimated SALT rather than using the $36,233.77 allowable amount, so it did not reach taxable income of $134,061.22."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model included the $1,632 refund in AGI and subtracted a nonexistent $5,050 personal exemption under an assumed TCJA sunset. The correct calculation uses $198,505 AGI, $64,443.77 of itemized deductions, and no personal exemption."
+us,scenario_078,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model did not apply the full $64,443.77 itemized deduction or did not use the resulting $134,061.22 taxable income. Applying the 2026 single brackets to that taxable income yields $24,772.69, not $31,316.32."
+us,scenario_078,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model ignored the $64,443.77 itemized deduction and fabricated a $12,000 child/dependent-care credit despite the household having no dependent or qualifying-care expense. CDCC is zero, and taxable income is determined using mortgage-interest and SALT itemization."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap and therefore omitted $26,233.77 of allowable SALT deduction. Its $42,034 also fails to follow from itemizing even its stated $38,210, showing that it did not apply the deductions and brackets consistently."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted amount implies that the model failed to use the full $36,233.77 SALT deduction and resulting $134,061.22 taxable income. No nonrefundable credit alters the bracket tax of $24,772.69."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model included the $1,632 refund in AGI and counted only $26,460 of property tax for SALT, omitting deductible Maryland income tax. The full SALT deduction is $36,233.77, bringing total itemized deductions to $64,443.77."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model did not derive the correct taxable income from the mortgage-interest and SALT deductions. Those deductions total $64,443.77 against $198,505 AGI, leaving $134,061.22 and tax of $24,772.69."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $1,632 refund to AGI and limited SALT to $26,460 of real-estate tax. It omitted deductible Maryland income tax that raises SALT to $36,233.77 and lowers taxable income to $134,061.22."
+us,scenario_078,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model counted only real-estate tax in the SALT deduction and omitted deductible Maryland income tax. Total SALT is $36,233.77, so itemized deductions are $64,443.77 rather than $54,670."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $41,500 answer is inconsistent with taxing the reference taxable income of $134,061.22 under the 2026 single brackets. It reflects failure to apply the $64,443.77 itemized deduction before computing regular income tax."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly included the refund in AGI, subtracted a restored personal exemption, and applied post-TCJA-sunset brackets. The operative calculation uses no personal exemption and applies the 2026 brackets to $134,061.22."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used restored 10/15/25/28 percent brackets and a personal exemption, neither of which belongs in this 2026 computation. It also included the refund in AGI and underestimated allowable SALT relative to $36,233.77."
+us,scenario_078,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed pre-TCJA brackets and subtracted a personal exemption instead of applying the operative 2026 schedule with no personal exemption. It also included the $1,632 refund in AGI and used an estimated SALT figure rather than $36,233.77."
+us,scenario_078,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model included the refund in AGI and treated the $26,460 property tax as the entire SALT deduction. It omitted deductible Maryland income tax, which raises SALT to $36,233.77 and total itemized deductions to $64,443.77."
+us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive tax calculation."
+us,scenario_078,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the $1,632 refund in AGI and omitted deductible Maryland income tax from SALT, counting only $26,460 of property tax. The correct SALT deduction is $36,233.77 and taxable income is $134,061.22."
+us,scenario_078,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite $134,061.22 of taxable income and provided no computation. Applying the 2026 single-filer brackets produces $24,772.69 of federal income tax before refundable credits."
+us,scenario_078,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model included the $1,632 refund in AGI and limited SALT to the $26,460 real-estate tax, omitting deductible Maryland income tax. SALT is $36,233.77, making taxable income $134,061.22 rather than $145,467."
+us,scenario_078,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model imposed an obsolete $10,000 SALT cap, included the refund in AGI, and ultimately used 2024 brackets instead of the operative 2026 parameters. The proper itemized deduction is $64,443.77 and the proper taxable income is $134,061.22."
+us,scenario_078,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model manufactured an AMT liability using an unsupported preference amount and offset, while also including the refund in AGI and omitting deductible Maryland income tax from SALT. The controlling calculation is regular tax on $134,061.22 after $64,443.77 of itemized deductions, with no such AMT addition."
 us,scenario_078,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_078,local_income_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model incorrectly treated Maryland county income tax as part of the requested local_income_tax variable and applied an assumed 3.03% average county rate to taxable income. This output excludes Maryland county tax and only captures the specifically listed modeled city taxes, none of which applies to this household."
 us,scenario_078,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived the exact $14,317.32 total but submitted $12,240.44 instead. Its numeric output discarded its own Social Security-plus-Medicare computation."
@@ -4637,30 +4870,31 @@ us,scenario_078,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals
 us,scenario_078,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model returned zero despite $198,505 of employee wages being subject to Social Security and Medicare taxes. It omitted both the $11,439 capped Social Security tax and the $2,878.32 uncapped Medicare tax."
 us,scenario_078,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model projected a $174,900 Social Security wage base instead of using the 2026 $184,500 base. This understated the Social Security component by $595.20 while its Medicare and Additional Medicare Tax treatment was correct."
 us,scenario_078,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model used a $10,453.20 Social Security amount corresponding to the obsolete $168,600 wage base rather than $11,439 on $184,500. Its stated components total $13,331.52, so the submitted $15,622.81 also contradicts its own arithmetic."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction and an exemption instead of the larger $54,670 Maryland itemized deduction. That left taxable income near $195,005 rather than $143,835 and overstated the tax."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invoked the federal SALT cap and described Maryland taxable income near $195,305 instead of deducting the full $28,210 mortgage interest and $26,460 property taxes allowed in the traced Maryland calculation. The correct $54,670 itemized deduction leaves $143,835 taxable."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived taxable income of $143,835 and tax of approximately $6,936.59, then submitted $9,755 instead. Its final value discarded its own Maryland bracket calculation."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model reduced the itemized deduction to an unsupported approximation of $30,000 instead of using $54,670, omitting much of the $26,460 property-tax deduction. It therefore taxed about $168,505 rather than $143,835 and then submitted a value inconsistent even with its stated $8,277.78 calculation."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model reduced the Maryland itemized deduction to about $38,210 instead of $54,670, effectively excluding $16,460 of the property taxes. It consequently used taxable income near $161,900 rather than $143,835."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model chose Maryland’s small standard deduction and an unphased personal exemption instead of the beneficial $54,670 itemized deduction. It also retained the $1,632 state-tax refund in the Maryland starting income, producing $194,537 of taxable income instead of $143,835."
-us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model reached the correct Maryland taxable income of $143,835 after the $54,670 itemized deduction but failed to apply the stated graduated brackets to that base. Those brackets yield $6,936.34, not the submitted $9,200."
-us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $2,400 standard deduction and $3,200 personal exemption while ignoring the larger $54,670 itemized deduction. It also included the $1,632 refund in Maryland AGI, leaving taxable income at $194,537 instead of $143,835."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $9,070 reflects taxation of a base far above the traced $143,835 taxable income. The model’s generic federal-AGI shortcut omitted the beneficial $54,670 Maryland itemized deduction."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model explicitly included a typical Maryland county or local income tax in an output that excludes local income tax. Only the Maryland state schedule applies here, producing $6,936.34 on $143,835 of taxable income."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified $143,835 of taxable income but misapplied Maryland’s graduated rate schedule. Applying the schedule to that base yields $6,936.34, not $6,886."
-us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model calculated the $6,936.34 state tax correctly and then added $4,602.72 of county tax. County tax belongs outside the requested state-income-tax output, so the submitted total improperly combines state and local liabilities."
-us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model started from $200,137 rather than Maryland adjusted gross income of $198,505, leaving $145,467 after itemization instead of $143,835. It also miscomputed the bracket tax on its stated base, since the extra $1,632 of taxable income cannot explain its $364.35 overstatement."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $9,720 reflects taxation of income near the pre-itemization base rather than $143,835. The model failed to apply the $54,670 Maryland itemized deduction comprising mortgage interest and property taxes."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Despite mentioning itemized deductions, the submitted $10,120 reflects a taxable base near gross income rather than $143,835. The calculation failed to subtract the full $54,670 Maryland itemized deduction before applying the state brackets."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model stated that it used allowable itemized deductions, but its $6,511.18 result understates the tax produced by Maryland’s schedule on $143,835. The correct graduated-bracket computation yields $6,936.34, with no additional personal exemption in the traced taxable-income figure."
-us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified itemization and no state credits but did not compute Maryland’s graduated schedule accurately. Tax on the resulting $143,835 taxable income is $6,936.34 rather than the rounded $7,025 estimate."
-us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $9,200 answer is consistent with taxing a base near adjusted gross income rather than the traced $143,835 taxable income. It omitted or materially reduced the $54,670 itemized deduction for mortgage interest and property taxes."
-us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used the $2,400 standard deduction instead of the beneficial $54,670 itemized deduction and retained the $1,632 refund in Maryland AGI. This inflated taxable income to $197,737 instead of $143,835."
-us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model imposed an unsupported $3,004 reduction on the $54,670 itemized deduction and then allowed a residual personal exemption. The traced Maryland computation deducts the full $54,670 from $198,505, leaving $143,835 taxable."
-us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric value for the requested output. It therefore failed the required structured-output contract without performing a usable Maryland tax calculation.
-us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model set the liability to zero despite $143,835 of Maryland taxable income. Applying Maryland’s positive graduated rates to that base produces $6,936.34 before refundable credits."
-us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the household as having no Maryland-specific listed deductions and used only a standard deduction, ignoring $28,210 of mortgage interest and $26,460 of property taxes. It also submitted $10,933 even though its own initial bracket arithmetic totaled $9,927.78."
-us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly began from federal taxable income and then subtracted a Maryland standard deduction instead of starting with Maryland adjusted gross income and taking the $54,670 itemized deduction. It also invented an unspecified $736 nonrefundable credit and used a rate schedule that does not match Maryland’s stated brackets."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model selected a small Maryland standard deduction and ignored the much larger $54,670 itemized deduction. The mortgage interest and property taxes reduce taxable income to $143,835, not approximately $195,005."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invoked the federal SALT cap and then based its estimate on approximately $195,305 of Maryland taxable income instead of deducting the full listed $54,670 of Maryland itemized deductions. Maryland taxable income is $143,835."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly derived $143,835 of taxable income and correctly computed approximately $6,936.59 from the Maryland brackets, but submitted $9,755 instead. Its submitted value contradicts its completed calculation."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model reduced the property-tax deduction and approximated total itemized deductions at only $30,000. The allowed mortgage-interest and property-tax deductions total $54,670, producing $143,835 of taxable income."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the itemized deduction as roughly $38,210 rather than $54,670, omitting $16,460 of the listed property taxes. It therefore used an overstated taxable-income base and also misstated the applicable upper marginal rate."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model chose the $2,400 standard deduction and a personal exemption while ignoring the beneficial $54,670 itemized deduction. It also retained the $1,632 state-tax refund in the Maryland income base, leading to taxable income of $194,537 instead of $143,835."
+us,scenario_078,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly reached $143,835 of Maryland taxable income but failed to apply the stated graduated rate schedule to that amount. That schedule yields $6,936.34, not $9,200."
+us,scenario_078,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a standard deduction and personal exemption instead of the $54,670 itemized deduction. It also left the state-tax refund in Maryland AGI, overstating taxable income as $194,537 rather than $143,835."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted estimate implies taxation of a base far above $143,835 and does not account for the $54,670 itemized deduction. Applying the Maryland schedule to the properly reduced base yields $6,936.34."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model included a typical Maryland county or local income tax in an output that expressly excludes local tax. Only the Maryland state schedule applies here, producing $6,936.34."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly identified taxable income of $143,835 but misapplied or approximated Maryland’s graduated brackets. Exact application of the schedule produces $6,936.34, not $6,886."
+us,scenario_078,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model explicitly added $4,602.72 of county tax to the correctly calculated $6,936.34 state tax. The requested variable excludes local and county income taxes."
+us,scenario_078,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $1,632 state-tax refund from the Maryland income starting point, leaving taxable income at $145,467. The correct Maryland base is $198,505 and falls to $143,835 after the $54,670 itemized deduction."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The $9,720 estimate implies that the model did not fully apply the $54,670 Maryland itemized deduction. That deduction reduces taxable income to $143,835 before the graduated schedule is applied."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"Despite mentioning itemized deductions, the submitted $10,120 is consistent with taxing nearly all wage income rather than subtracting the full $54,670. The correct taxable-income base is $143,835."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model introduced a personal-exemption adjustment beyond the traced $54,670 deduction or otherwise understated the taxable base. The applicable deduction leaves $143,835 taxable, whose scheduled tax is $6,936.34."
+us,scenario_078,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model said it used the available itemized deductions but did not calculate Maryland’s brackets exactly. On the resulting $143,835 taxable income, the schedule yields $6,936.34 rather than $7,025."
+us,scenario_078,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $9,200 answer is consistent with taxing a base substantially above the correct $143,835 and therefore failing to give full effect to the $54,670 itemized deduction. The proper deduction and Maryland schedule yield $6,936.34."
+us,scenario_078,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used only a $2,400 standard deduction and ignored the beneficial $54,670 itemized deduction for mortgage interest and property taxes. It also retained the $1,632 refund in Maryland AGI, producing a severely overstated taxable base."
+us,scenario_078,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model imposed an unsupported $3,004 reduction on Maryland itemized deductions and retained the $1,632 refund in Maryland AGI. The traced itemized deduction is the full $54,670, leaving taxable income of $143,835."
+us,scenario_078,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value for the requested output. This is a missing-output contract failure rather than a substantive tax calculation.
+us,scenario_078,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite $143,835 of Maryland taxable income after the allowed itemized deductions. Applying the state’s graduated schedule produces a positive $6,936.34 liability."
+us,scenario_078,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model retained the $1,632 state-tax refund in Maryland AGI, producing taxable income of $145,467 instead of $143,835. It also applied 4.75% to all income above $3,000 rather than using the higher graduated brackets above $100,000."
+us,scenario_078,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the household as having no Maryland-specific listed deductions and therefore ignored $28,210 of mortgage interest and $26,460 of property taxes. Those itemized deductions total $54,670 and reduce taxable income to $143,835."
+us,scenario_078,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly started from federal taxable income, used a fabricated Maryland bracket structure, and subtracted an unidentified $736 nonrefundable credit. Maryland instead applies its actual graduated schedule to $143,835 of state taxable income, with no such traced credit."
 us,scenario_078,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_079,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model ignored that the head receives SSI and therefore belongs to Arizona's automatic SSI-recipient Medicaid category. It incorrectly demanded an additional eligibility pathway or coverage input even though SSI receipt itself establishes Medicaid eligibility.
 us,scenario_079,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model assumed that the head had completed the 24-month SSDI waiting period even though no disability-entitlement duration was listed. Constant disability and SSDI receipt during 2026 do not supply that missing prior-period history, so the 58-year-old head remains ineligible."
@@ -4683,93 +4917,97 @@ us,scenario_079,head_medicare_eligible,minimax-m3,llm_error,age_disability,False
 us,scenario_079,head_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model misread constant facts throughout the tax year as proof that the head had completed Medicare’s 24-month disability waiting period. Constancy establishes only the 2026 status, not prior entitlement history, so the age-58 head is not eligible."
 us,scenario_079,local_income_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_079,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the household's $881 monthly SSI from SNAP gross income and used only the $630 monthly SSDI. SNAP gross income is $1,511 and net income after $354 of deductions is $1,157, not the model's $395–$505 range."
-us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model treated home and vehicle equity as disqualifying resources despite the residence and applicable vehicle value being excluded and countable assets being only $1,045.29. The household passes the asset test and is categorically eligible through SSI receipt and TANF non-cash eligibility."
-us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted $881 per month of SSI and calculated the benefit from SSDI alone. It also replaced its own formula result with an unsupported $351.50 monthly figure instead of using $1,157 of net income against the applicable maximum allotments."
-us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model omitted the $881 monthly SSI from gross income and then asserted a net-income and benefit figure inconsistent with its own deductions. The SNAP budget uses $1,511 of gross income, $354 of deductions, and $1,157 of net income."
-us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,The model based the calculation only on $630 monthly SSDI and omitted $881 monthly SSI. Its stated $283 benefit also does not follow from its claimed net income near $400 and maximum allotment of $536.
-us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model excluded $881 per month of SSI from gross income and therefore reduced net income to $400. The correct SNAP budget starts with $1,511 of gross income and applies total deductions of $354, yielding $1,157."
-us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and invented shelter costs from the mortgage balance even though no mortgage payment, interest, rent, or utilities were listed. A mortgage balance is not a deductible shelter expense, and the actual budget has $1,157 of net income."
-us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"The model applied an obsolete vehicle-equity exclusion calculation and counted $5,050 of vehicle value as a resource. PolicyEngine excludes the applicable vehicle value, leaving $1,045.29 of countable assets, so the household passes the resource test."
-us,scenario_079,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included SSDI and SSI but allowed only $224 of monthly deductions, producing $1,211 of net income. The SNAP computation applies $354 of deductions and therefore uses $1,157 of net income; it also uses the actual 2026 allotment changes rather than a constant $535 maximum."
-us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of housing expenses as eliminating SNAP benefits. Shelter expenses affect deductions, not basic eligibility, and this household qualifies through its income/resource tests and categorical eligibility from SSI and TANF non-cash eligibility."
-us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $17,220 annual income and concluded that the expected contribution exhausted the normal allotment. The monthly budget instead has $1,511 gross income, $354 of deductions, and $1,157 net income, leaving benefits of $198.90 or $212.64 rather than the minimum floor."
-us,scenario_079,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model overstated gross income as $18,072 and allowed only $2,712 of annual deductions. The trace uses $1,511 monthly gross income and $354 monthly deductions, so the expected contribution is based on $1,157 rather than $1,280 of monthly net income."
-us,scenario_079,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model stated the correct general income components but did not apply the traced $354 monthly deduction total. The resulting SNAP formula must use $1,157 of net income and the two applicable 2026 maximum allotments, yielding $2,428.02 annually."
-us,scenario_079,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model effectively awarded a near-maximum allotment without subtracting the household's expected contribution. SNAP subtracts about $347, equal to 30% of $1,157 net income, from each applicable monthly maximum."
-us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no parseable SNAP value or explanation.
-us,scenario_079,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the lack of dependents and shelter costs as grounds for a zero benefit. Two disabled adults can qualify, and this household passes the net-income and asset tests while also receiving categorical eligibility through SSI and TANF non-cash eligibility."
-us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the explicitly listed SSDI, disability, assets, and assumed program take-up, then demanded additional SNAP-enabling facts. The supplied facts establish eligibility and support a positive benefit calculated from $1,157 of net income."
-us,scenario_079,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model included both SSDI and SSI but deducted only $230 per month, producing $1,281 of net income. PolicyEngine applies $354 of deductions, leaving $1,157 and reducing the expected contribution from $385 to about $347."
-us,scenario_079,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model awarded the full maximum allotment and failed to subtract 30% of net income. The household has $1,157 of SNAP net income, creating an expected contribution of about $347 per month."
-us,scenario_079,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $163 monthly estimate reflects excessive net income from incomplete deductions. Total deductions are $354 per month, leaving $1,157 of net income and normal allotments of $198.90 and $212.64 under the changing 2026 maxima."
-us,scenario_079,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model understated the benefit by using a larger expected contribution than the traced budget supports. SNAP net income is $1,157 after $354 of deductions, so the contribution is about $347 and the monthly allotments are $198.90 and $212.64."
-us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly asserted that $7,560 of annual SSDI could exceed the two-person SNAP income limits and returned zero without applying the disabled-household rules. The household passes both income tests and categorical eligibility, and its $1,157 monthly net income produces a positive allotment."
-us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted $881 per month of SSI and calculated SNAP from SSDI alone. It therefore used annual net income of only $4,728 instead of the traced monthly gross income of $1,511 and monthly net income of $1,157."
-us,scenario_079,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model invented an SSI amount from an estimated couple rate rather than using the engine's $881 monthly SSI receipt, then allowed only $230 of deductions. The correct budget uses $1,511 gross income and $354 of deductions, producing $1,157 net income."
-us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model omitted $881 of monthly SSI and computed the expected contribution from only $395 of net income. The correct contribution is approximately 30% of $1,157, not 30% of an SSDI-only budget."
-us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable SNAP value or explanation.
-us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted $881 per month of SSI and used only SSDI in gross income. That omission reduced its net income to $395 instead of the traced $1,157 and overstated the allotment."
-us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,The model disregarded the prompt's explicit instruction to assume program take-up and treated unlisted enrollment as a reason to return zero. Eligibility and assumed take-up produce the full calculated annual SNAP benefit.
-us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,The model omitted the household's $881 monthly SSI and incorrectly applied the $35 medical threshold once per year rather than monthly. It also used an overstated $598 maximum allotment instead of the applicable $546 and $558.24 amounts.
-us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model reversed the effect of SSI receipt: SSI supports categorical SNAP eligibility rather than replacing SNAP with a bundled benefit. The household separately receives a positive SNAP allotment based on its $1,157 net income."
+us,scenario_079,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the household's $881 monthly SSI from SNAP gross income and calculated from only $630 of SSDI. Its unexplained reduction from a computed $5,010 to $4,560 also abandons its own stated deduction formula."
+us,scenario_079,snap,claude-haiku-4.5,llm_error,asset_resource,False,"The model counted vehicle equity and speculated about home equity even though the applicable SNAP treatment excludes the home and does not produce excess countable vehicle resources here. Countable assets are $1,045.29, so the household passes the asset test."
+us,scenario_079,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI from gross income, using only the $630 SSDI amount. It then replaced its own approximately $5,046 calculation with $4,218 without a valid SNAP computation."
+us,scenario_079,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,The model omitted $881 monthly SSI and based the benefit on only $630 of SSDI. It also reversed the effect of the medical deduction by claiming that a lower net income led to a lower benefit.
+us,scenario_079,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,The model excluded $881 monthly SSI from SNAP gross income and used only $630 of SSDI. Its stated $400 net income cannot yield a $283 allotment against a $536 maximum under the 30% contribution formula.
+us,scenario_079,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI from gross income, reducing the SNAP budget to $630 before deductions instead of $1,511. The correct engine budget applies $354 of deductions and reaches $1,157 net income, not $400."
+us,scenario_079,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $881 monthly SSI and invented unlisted mortgage shelter costs from the mortgage balance, contrary to the instruction that unlisted expenses are zero. It also used an incorrect $374 maximum allotment for a two-person household."
+us,scenario_079,snap,deepseek-v4-pro,llm_error,asset_resource,False,"The model applied an obsolete vehicle-equity exclusion calculation and counted $5,050 of the vehicle as a resource. Under the applicable SNAP resource treatment, countable household assets are $1,045.29 and pass the test."
+us,scenario_079,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model correctly included SSDI and SSI but deducted only the standard and $26 excess-medical amounts, producing $1,211 rather than the engine's $1,157 net income after $354 of total deductions. It also used a flat $535 maximum instead of the applicable 2026 parameter periods."
+us,scenario_079,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated the absence of housing expenses as making the SNAP benefit zero. Shelter costs are not required for eligibility, and SSI receipt supplies categorical eligibility while the income and asset tests are also satisfied."
+us,scenario_079,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $17,220 of annual income instead of the engine's $18,132 annualized gross income and failed to apply the full $354 monthly deductions. The normal allotment remains well above the minimum-benefit floor, so reducing the award to $276 is incorrect."
+us,scenario_079,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used $18,072 rather than $18,132 of annualized SSDI-plus-SSI income and allowed only $2,712 of annual deductions rather than the engine's $4,248. It also used an inflated $6,720 annual maximum allotment."
+us,scenario_079,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used $18,096 of gross annual income instead of $18,132 and did not derive the engine's $1,157 monthly net income after $354 of deductions. That unsupported net-income estimate understates the annual allotment."
+us,scenario_079,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer is consistent with awarding close to the maximum allotment while failing to subtract the household's expected contribution. Net income is $1,157 monthly, and approximately 30% must be deducted from the two-person maximum."
+us,scenario_079,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required submission contract."
+us,scenario_079,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of dependents and shelter costs as precluding SNAP. The two disabled adults qualify through SSI categorical eligibility and the income and asset tests, with shelter expenses unnecessary for a positive allotment."
+us,scenario_079,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the explicitly listed SSDI, disability, assets, and assumed program take-up and asserted that SNAP-enabling facts were absent. SSI receipt establishes categorical eligibility, and the resulting income budget produces a positive benefit."
+us,scenario_079,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model included both income sources but allowed only $230 of monthly deductions, yielding $1,281 net income. PolicyEngine applies $354 of deductions, yielding $1,157 and a larger normal allotment across the applicable monthly maximums."
+us,scenario_079,snap,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model awarded the full maximum allotment and failed to subtract the expected contribution from $1,157 of net income. SNAP requires an approximately 30% net-income contribution, leaving about $199–$211 per month rather than $546."
+us,scenario_079,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's $163 monthly estimate reflects an overstated net-income contribution. The engine applies $354 in deductions to $1,511 gross income, producing $1,157 net income and annual benefits of $2,428.02 after the changing monthly maximums."
+us,scenario_079,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model understated the allotment by using an incorrect countable-income or deduction result. Gross income is $1,511 monthly, total deductions are $354, and the resulting $1,157 net income produces $2,428.02 annually."
+us,scenario_079,snap,grok-4.3,llm_error,thresholds_rates,False,"The model asserted that $630 monthly SSDI exceeded the SNAP limit, although total gross income of $1,511 is only 86% of the applicable poverty guideline. The household passes both gross and net income tests and receives a positive allotment."
+us,scenario_079,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,The model omitted $881 monthly SSI and computed the budget from SSDI alone. Its annual standard and medical deductions therefore operate on the wrong gross-income base and substantially overstate the benefit.
+us,scenario_079,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model included SSI but allowed only about $235 of deductions, leaving roughly $1,271 net income. The engine applies $354 in deductions and reaches $1,157, which raises the normal allotment to the $2,428.02 annual total."
+us,scenario_079,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model invented an SSI couple-rate calculation and a $20 exclusion instead of using the engine-derived $881 SSI amount. It then allowed only $230 of deductions rather than $354, producing $1,240 instead of $1,157 net income."
+us,scenario_079,snap,inkling,llm_error,taxable_income_or_deductions,False,The model omitted the $881 monthly SSI and calculated SNAP solely from $630 of SSDI. This reduced the expected contribution by roughly $264 per month and greatly overstated the allotment.
+us,scenario_079,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required submission contract."
+us,scenario_079,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model omitted $881 monthly SSI and reduced only the $630 SSDI amount by standard and medical deductions. The correct gross-income base is $1,511 monthly and the engine's net income is $1,157."
+us,scenario_079,snap,minimax-m3,llm_error,categorical_eligibility,False,The model disregarded the prompt's explicit instruction to assume program take-up when required and demanded evidence of enrollment. Eligibility and assumed take-up produce a positive SNAP award; prior enrollment is not required.
+us,scenario_079,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,The model omitted $881 monthly SSI and calculated net income from only $630 of SSDI. That omission sharply reduces the 30% expected contribution and inflates the annual benefit.
+us,scenario_079,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted the $881 monthly SSI and incorrectly applied the $35 medical threshold once per year rather than monthly. It also used an inflated $598 monthly maximum allotment, compounding the overstatement."
+us,scenario_079,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated SSI receipt as replacing or barring SNAP. SSI instead supports categorical SNAP eligibility, and the household receives a separately calculated positive allotment."
 us,scenario_079,spouse_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability and IHS coverage as the only relevant facts and failed to account for the spouse's engine-computed $5,286 of SSI receipt. That receipt places the spouse in Arizona's SSI-recipient category, which automatically confers Medicaid eligibility without further income verification."
 us,scenario_079,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted the spouse's disability flag directly into Medicare eligibility and incorrectly inferred an SSDI-linked entitlement. Disability alone is not a Medicare pathway, and the spouse has no listed SSDI receipt or other qualifying basis before age 65."
 us,scenario_079,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,The model identified the 24-month SSDI-entitlement requirement but then ignored it and treated disability as sufficient. It also invented SSDI-equivalent status for the spouse even though only the head receives Social Security disability income and unlisted benefit receipt is zero.
-us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"The model projected a $1,450.60 monthly eligible-couple rate instead of applying the 2026 $1,491 amount. With $610 of countable monthly income, the correct subtraction yields $881 per month, not $840.70."
-us,scenario_079,ssi,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly tested the head’s annual SSDI against an individual SSI limit and also falsely stated that $7,560 annually exceeds roughly $943 monthly. Both spouses are eligible individuals, so the eligible-couple rules and spousal deeming produce $610 in monthly countable income against the $1,491 couple amount."
-us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model abandoned its initial couple-rate estimate and substituted an unsupported $1,940 monthly federal benefit rate. The applicable 2026 eligible-couple amount is $1,491, leaving $881 monthly after countable income rather than $1,330."
-us,scenario_079,ssi,claude-opus-4.8,llm_error,other,False,"The model used an incorrect $1,538 monthly couple amount and then submitted $7,212 despite its own calculation producing $11,136. Applying the actual $1,491 amount and subtracting $610 yields $10,572 annually."
-us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"The model’s submitted $6,072 does not follow from its own figures: $17,868 minus $7,320 equals $10,548. It also used the wrong annual couple maximum; the applicable $17,892 maximum less $7,320 equals $10,572."
-us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the 2026 eligible-couple amount as $1,450 monthly instead of $1,491. The resource and income treatment was otherwise aligned, but the wrong benefit rate understated SSI by $41 monthly."
-us,scenario_079,ssi,claude-sonnet-5,llm_error,other,False,"The model explicitly concluded that the couple passes the resource test and receives a positive SSI payment, then submitted 0 instead of its calculated amount. Its calculation also used an unsupported $1,927 monthly couple rate rather than the applicable $1,491 rate."
-us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $1,485 monthly eligible-couple rate instead of $1,491. That six-dollar monthly understatement produced $10,500 rather than $10,572 annually."
-us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the $1,415 monthly couple rate rather than the 2026 $1,491 amount. Subtracting the correctly identified $610 of countable income from $1,491 yields $881 monthly."
-us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the couple’s countable income exceeds the federal benefit rate. Countable income is $610 monthly, below the $1,491 eligible-couple amount, leaving an $881 monthly benefit."
-us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used a $16,980 annual maximum, equivalent to the outdated $1,415 monthly couple rate. The 2026 annual eligible-couple amount is $17,892, from which $7,320 of countable income leaves $10,572."
-us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used a projected annual couple maximum of $17,832 instead of $17,892. Its income subtraction was correct, but the $60 understatement of the maximum carried directly into the answer."
-us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model implicitly used an annual eligible-couple maximum of $17,856, which is $36 below the applicable $17,892. Deducting the same $7,320 of countable income therefore understated annual SSI by $36."
-us,scenario_079,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer implies an annual couple maximum of $17,808 after adding the stated $7,320 countable income back to the benefit. The applicable maximum is $17,892, so the model understated SSI by $84."
-us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, so the required numeric value was missing."
-us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model counted the vehicle toward SSI resources even though the household vehicle is excluded. Countable resources are $1,045.29, below the couple resource limit, so the resource test does not eliminate either spouse."
-us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated unlisted prior SSI receipt as necessary despite the prompt requiring program take-up and providing both spouses’ blindness and disability. Those facts establish the categorical pathway, and the supplied income and asset facts are sufficient to compute the benefit."
-us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a projected $1,487 monthly couple maximum instead of $1,491. Its $610 countable-income figure was correct, but the four-dollar monthly rate error understated annual SSI by $48."
-us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a $1,486 monthly couple payment standard instead of $1,491. Subtracting $610 from the correct standard gives $881 monthly rather than $876."
-us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"The model incorrectly treated SSDI as exceeding the eligible couple’s SSI limit. After the income exclusion, only $610 is countable each month against the $1,491 couple amount, leaving $881 payable."
-us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used an estimated $1,486 monthly eligible-couple rate rather than $1,491. Its otherwise correct $610 income reduction therefore produced $876 instead of $881 monthly."
-us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a $1,450 monthly couple rate instead of the 2026 $1,491 amount. With $610 in monthly countable income, the benefit is $881 rather than $840."
-us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, so the required numeric value was missing."
-us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model treated prior SSI receipt as a required input even though the task asks for the benefit under assumed take-up. Both spouses satisfy the blind-or-disabled criterion, pass the resource test, and receive the amount determined by the couple computation."
-us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"After correcting its initial annual-versus-monthly exclusion error, the model still used an estimated $1,450 monthly couple rate instead of $1,491. The correct rate less $610 of monthly countable income produces $10,572 annually."
-us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model cut the couple’s countable income in half to $3,780 annually and deducted only that amount from the combined benefit. Spousal deeming splits the income for individual records but does not halve the household reduction: total countable income is $7,320 annually, producing $10,572 for the couple."
-us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly imposed an earned-income or taxable-income filing requirement on the increased excise tax credit. The two eligible spouses receive $25 each despite having only exempt SSDI, yielding $50."
-us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Arizona has no refundable individual credit and tied credit availability to taxable income. Arizona’s refundable increased excise tax credit provides $25 for each of the two spouses, independent of positive Arizona taxable income."
-us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly identified the increased excise tax credit as $25 per person and explicitly computed $50 for the two-person joint return, but then submitted $200 without any supporting component. It failed to carry its own $50 computation into the final value."
-us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly treated positive taxable income or a filing threshold as necessary for the increased excise tax credit. The eligible joint-filing couple receives $25 per adult even though their SSDI is exempt, totaling $50."
-us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly required earnings or qualifying dependents for all Arizona refundable credits. The increased excise tax credit instead pays $25 for each of these two eligible adults, totaling $50."
-us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model incorrectly classified the increased excise tax credit as nonrefundable and treated exempt SSDI and zero wages as disqualifying. That credit is refundable and supplies $25 per eligible spouse, for $50 total."
-us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model considered property-tax-credit eligibility but omitted the separate refundable increased excise tax credit. The two eligible adults on the Arizona joint return generate $25 each, totaling $50."
-us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,The model wrongly treated dependent children or property-tax-credit eligibility as necessary for any refund. Arizona’s increased excise tax credit applies to this two-adult joint tax unit and yields $50.
-us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer omits Arizona’s increased excise tax credit. Residence, joint filing status, and the two-person tax-unit size produce $25 per adult, or $50."
-us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model incorrectly concluded that no refundable credit applied and omitted the increased excise tax credit eligibility pathway. The two eligible spouses generate a $50 credit.
-us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model omitted Arizona’s refundable increased excise tax credit. At $25 for each of the two eligible adults, the correct computation yields $50."
-us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model failed to apply the increased excise tax credit to the Arizona married-joint tax unit. Two eligible adults at $25 each produce $50.
-us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model’s blanket denial of Arizona refundable credits omits the increased excise tax credit. The household’s two eligible adults qualify for $25 each, totaling $50."
-us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model wrongly used zero taxable income to exclude Arizona refundable credits. The increased excise tax credit remains available and pays $25 for each spouse, producing $50."
-us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to trigger Arizona’s increased excise tax credit from the household’s residence, joint filing status, and two-adult tax-unit size. Those facts yield $50."
-us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly looked for earned income or children and omitted the increased excise tax credit, which requires neither here. The two eligible adults receive $25 each, totaling $50."
-us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required tax liability, earned income, or qualifying children for an Arizona refundable credit. The increased excise tax credit is refundable under these facts and equals $25 per spouse, or $50."
-us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model overlooked that Arizona residence, married-joint status, and a tax-unit size of two trigger the increased excise tax credit. The resulting credit is $50."
-us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly made earnings, children, or another special circumstance prerequisites for every refundable Arizona credit. The increased excise tax credit pays this eligible two-adult joint unit $50."
-us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly treated zero state tax liability as eliminating refundable credits. Arizona’s increased excise tax credit is payable beyond liability and supplies $25 per spouse, totaling $50."
-us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly required qualifying children or positive tax liability for a refundable Arizona credit. The increased excise tax credit applies to the two eligible adults and totals $50.
-us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model stopped after evaluating a supposed Arizona EITC and omitted the independent increased excise tax credit. That credit produces $25 for each spouse, totaling $50, regardless of federal EITC eligibility."
-us,scenario_079,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model incorrectly used zero state tax liability to rule out a refundable credit and omitted the increased excise tax credit. The two eligible spouses receive $25 each, for $50."
-us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model limited its analysis to property-tax refunds and credits requiring earnings or other inputs, omitting the increased excise tax credit. That separate credit needs none of the cited expenses and pays $50 to this two-adult joint unit."
-us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,The model incorrectly treated being childless and having zero tax liability as disqualifying for all Arizona refundable credits. The increased excise tax credit applies to the two spouses and yields $50.
-us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Arizona has no state income tax. It consequently omitted Arizona’s refundable increased excise tax credit, which provides $25 per eligible spouse and totals $50."
-us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly required Arizona taxable income or qualifying dependents for a refundable credit. The increased excise tax credit applies independently and pays $25 for each of the two adults, totaling $50."
-us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Arizona’s increased excise tax credit without applying its residence, filing-status, and tax-unit-size rules. Those rules produce $25 per spouse, or $50 total."
+us,scenario_079,ssi,claude-fable-5,llm_error,thresholds_rates,False,"The model substituted an estimated $1,450.60 monthly couple rate for the applicable $1,491 amount. Subtracting the correctly computed $610 countable income from $1,491 yields $881 monthly and $10,572 annually."
+us,scenario_079,ssi,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model incorrectly tested the head's annual SSDI against an individual SSI limit and also falsely stated that $7,560 annually exceeds roughly $943 monthly. Both spouses are eligible individuals under the couple computation, and spousal deeming plus the income exclusions produces $610 of monthly countable income, leaving a positive benefit."
+us,scenario_079,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model abandoned its initial couple-rate calculation and used an unsupported $1,940 monthly federal benefit rate. The applicable eligible-couple amount is $1,491, so the $610 monthly countable income leaves $881 rather than $1,330."
+us,scenario_079,ssi,claude-opus-4.8,llm_error,other,False,"The model's stated calculation produces $11,136, but it submitted $7,212 without any computation supporting that value. The applicable calculation is $1,491 minus $610 per month, producing $10,572 annually."
+us,scenario_079,ssi,claude-opus-5,llm_error,other,False,"The model stated an annual maximum of $17,868 and countable income of $7,320, whose difference is $10,548, but then submitted $6,072. It also used the wrong monthly couple amount: $1,491 minus $610 equals $881 monthly and $10,572 annually."
+us,scenario_079,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly handled categorical eligibility, resources, and the $610 monthly countable income but estimated the couple amount as $1,450. The applicable $1,491 amount raises the monthly benefit from its $840 calculation to $881."
+us,scenario_079,ssi,claude-sonnet-5,llm_error,other,False,"The model concluded that the household was eligible and calculated a positive $15,804 benefit, then submitted zero in direct contradiction to its reasoning. Its positive calculation also used an unsupported $1,927 couple rate instead of $1,491."
+us,scenario_079,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $1,485 monthly couple amount instead of $1,491. With the correctly identified $610 countable income, the benefit is $881 per month rather than $875."
+us,scenario_079,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used the $1,415 couple rate rather than the applicable 2026 amount of $1,491. Subtracting $610 from $1,491 produces $881 monthly and $10,572 annually."
+us,scenario_079,ssi,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the SSDI-derived countable income exceeds the couple benefit standard. Monthly countable income is $610, which is below the $1,491 eligible-couple amount and leaves $881 payable."
+us,scenario_079,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model used an outdated $16,980 annual maximum, equivalent to $1,415 monthly. The applicable maximum is $17,892 annually, and subtracting $7,320 of annual countable income yields $10,572."
+us,scenario_079,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model projected a $17,832 annual couple maximum, $60 below the applicable $17,892 maximum. Its countable-income calculation was correct, so this rate error understated annual SSI by exactly $60."
+us,scenario_079,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model implicitly used a $17,856 annual couple maximum after subtracting $7,320 of countable income. The applicable maximum is $17,892, producing $10,572 rather than $10,536."
+us,scenario_079,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted amount implies an annual couple maximum of $17,808 after adding back the correctly excluded-income result of $7,320. The applicable maximum is $17,892, so the model understated SSI by $84."
+us,scenario_079,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output. The required numeric result was $10,572."
+us,scenario_079,ssi,gpt-5.4-mini,llm_error,asset_resource,False,"The model counted the vehicle toward SSI resources even though one household vehicle is excluded. Countable resources are only $1,045.29, below the couple limit, so the resource test does not eliminate either spouse."
+us,scenario_079,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated unlisted SSI receipt as a reason to assign zero despite the prompt's take-up assumption and the listed blindness and disability of both spouses. Those facts establish the aged, blind, or disabled criterion, while the supplied income and asset facts are sufficient to calculate a positive benefit."
+us,scenario_079,ssi,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used a projected $1,487 monthly couple maximum instead of $1,491. With $610 of monthly countable income, the correct payment is $881 monthly rather than $877."
+us,scenario_079,ssi,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a $1,486 monthly couple standard instead of $1,491. Its $610 countable-income figure was correct, but the proper subtraction yields $881 monthly rather than $876."
+us,scenario_079,ssi,grok-4.3,llm_error,thresholds_rates,False,"The model wrongly treated SSDI as exceeding the couple's SSI payment standard. After exclusions and spousal deeming, countable income is $610 per month, below the $1,491 eligible-couple amount."
+us,scenario_079,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used an estimated $1,486 monthly couple rate instead of $1,491. Subtracting the correctly calculated $610 countable income leaves $881 monthly, not $876."
+us,scenario_079,ssi,grok-4.6,llm_error,thresholds_rates,False,"The model correctly excluded the home and vehicle and calculated $610 of monthly countable income, but used a $1,486 couple rate. The applicable $1,491 rate produces $881 monthly and $10,572 annually."
+us,scenario_079,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used a $1,450 monthly couple rate rather than the applicable $1,491 amount. With $610 of countable income, monthly SSI is $881 rather than $840."
+us,scenario_079,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI output. The required numeric result was $10,572."
+us,scenario_079,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the absence of listed SSI receipt as grounds for zero, ignoring the instruction to assume program take-up. Both spouses satisfy the blind or disabled criterion, pass the resource test, and receive the positive amount generated after income counting."
+us,scenario_079,ssi,qwen-3.7-max,llm_error,thresholds_rates,False,"The model ultimately applied the $20 exclusion correctly as $240 annually but used an estimated $1,450 monthly couple amount. The applicable $1,491 amount minus $610 of monthly countable income yields $10,572 annually."
+us,scenario_079,ssi,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model reduced the couple benefit by only $3,780, effectively assigning only half of the SSDI-derived income to the combined computation. Spousal deeming splits the $7,560 unearned income between the spouses for individual eligibility, but total household countable income remains $7,320 annually, or $610 monthly."
+us,scenario_079,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly imposed an income or earnings requirement on Arizona's increased excise tax credit. The eligible married couple receives $25 per adult despite having only SSDI, producing $50."
+us,scenario_079,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that Arizona has no refundable individual credit and treated zero state-taxable income as disqualifying. Arizona's refundable increased excise tax credit provides this two-adult tax unit $50.
+us,scenario_079,state_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model correctly identified the increased excise tax credit as $25 for each of two adults and explicitly derived $50, but then submitted $200. No additional refundable credit in its reasoning supports the extra $150."
+us,scenario_079,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly required taxable income or satisfaction of an income-tax filing threshold for the increased excise tax credit. The qualifying married Arizona couple receives $25 per adult even with no Arizona taxable income.
+us,scenario_079,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model treated earnings and qualifying dependents as prerequisites for all Arizona refundable credits. Neither is required for this couple's $50 increased excise tax credit.
+us,scenario_079,state_refundable_credits,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model incorrectly classified the increased excise tax credit as nonrefundable and tied refundable-credit eligibility to wages or taxable income. The increased excise tax credit is refundable and awards this two-adult couple $50.
+us,scenario_079,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model considered the property tax credit but omitted the separate refundable increased excise tax credit. The couple's failure to meet the property tax credit's age or SSI conditions does not prevent the $50 excise credit.
+us,scenario_079,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model focused on dependent children and property tax credits while omitting Arizona's increased excise tax credit. This credit pays $25 for each adult in the qualifying married tax unit, totaling $50."
+us,scenario_079,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the Arizona increased excise tax credit from its eligibility assessment. The reported Arizona residence, joint filing status, and two-adult unit produce a $50 refundable credit."
+us,scenario_079,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable state credit applied, omitting Arizona's increased excise tax credit. The two eligible adults generate $25 each, totaling $50."
+us,scenario_079,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model failed to apply Arizona's refundable increased excise tax credit. The qualifying married tax unit receives $25 for each of its two adults, totaling $50."
+us,scenario_079,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model omitted the increased excise tax credit when screening Arizona refundable credits. This married two-adult household qualifies for $50.
+us,scenario_079,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model's blanket denial of Arizona refundable-credit eligibility missed the increased excise tax credit. Applying its $25-per-adult amount yields $50.
+us,scenario_079,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model incorrectly used zero taxable income to exclude all Arizona refundable credits. The increased excise tax credit remains available and supplies $25 per adult, or $50."
+us,scenario_079,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model failed to recognize that the listed Arizona residence and two-adult joint tax unit trigger the increased excise tax credit. That refundable credit equals $50.
+us,scenario_079,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly treated earned income or children as necessary supporting facts for an Arizona refundable credit. The increased excise tax credit requires neither here and provides $25 per adult, totaling $50."
+us,scenario_079,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly required tax liability, earned income, or qualifying children for every Arizona refundable credit. Arizona's increased excise tax credit is refundable without those conditions and gives this couple $50."
+us,scenario_079,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model overlooked the facts that generate Arizona's increased excise tax credit: Arizona residence, married-joint status, and two adults. Those facts yield a $50 refundable credit."
+us,scenario_079,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly conditioned all Arizona refundable credits on earnings, children, or another special circumstance. The increased excise tax credit applies to this qualifying two-adult joint return and equals $50."
+us,scenario_079,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model wrongly treated positive state tax liability as necessary for the refundable credit. Arizona's increased excise tax credit can exceed liability and awards this couple $50.
+us,scenario_079,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly relied on zero liability and the absence of children to deny all refundable credits. The increased excise tax credit requires neither positive liability nor qualifying children here and totals $50.
+us,scenario_079,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model limited its analysis to a state EITC and omitted Arizona's separate increased excise tax credit. A zero federal EITC does not eliminate the $50 excise credit for this two-adult joint tax unit.
+us,scenario_079,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model incorrectly used zero state tax liability to rule out refundable credits. The increased excise tax credit is payable independently of liability and equals $50 for the two adults.
+us,scenario_079,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model considered property-tax and earned-income-based credits but omitted Arizona's increased excise tax credit. That credit does not require listed rent, property tax, or earnings and gives the couple $50."
+us,scenario_079,state_refundable_credits,kimi-k3,llm_error,categorical_eligibility,False,The model incorrectly treated children and positive tax liability as necessary for an applicable Arizona refundable credit. The increased excise tax credit applies to this childless two-adult joint return and totals $50.
+us,scenario_079,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model made the foundational error that Arizona has no state income tax. Arizona has an individual income tax and a refundable increased excise tax credit, which supplies this couple $50."
+us,scenario_079,state_refundable_credits,ox-alpha,llm_error,categorical_eligibility,False,"The model stopped after rejecting the age-based property tax credit and omitted the separate increased excise tax credit. The couple receives that credit at $25 per adult, totaling $50."
+us,scenario_079,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly treated taxable income and qualifying dependents as necessary for all Arizona refundable credits. The increased excise tax credit requires neither for this qualifying couple and equals $50.
+us,scenario_079,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted Arizona's increased excise tax credit from its categorical eligibility determination. The married two-adult Arizona tax unit qualifies for $25 per adult, totaling $50."
 us,scenario_080,federal_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_080,federal_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_080,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -4790,134 +5028,143 @@ us,scenario_080,local_income_tax,glm-5.2,parse_contract_failure,missing_output,F
 us,scenario_080,payroll_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_080,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_080,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_080,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached zero net income and categorical eligibility but used a guessed flat maximum of $293 per month. The applicable monthly maximum changes from $298 to $304.68 during the year, producing $3,596.04 when annualized."
-us,scenario_080,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied the ordinary $2,750 resource limit and incorrectly added financial assistance to resources. Pennsylvania TANF non-cash categorical eligibility governs, and the household's assets are below the applicable categorical limits."
-us,scenario_080,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached maximum-allotment eligibility but substituted an approximate $293 monthly maximum. The 2026 computation annualizes monthly maxima of $298 through $304.68 to $3,596.04."
-us,scenario_080,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model imposed the ordinary elderly-or-disabled federal resource limit on a categorically eligible Pennsylvania household. TANF non-cash categorical eligibility allows the household's $19,827.65 of assets under Pennsylvania's limits."
-us,scenario_080,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model replaced the allotment formula with an unsupported range and selected $218.50 per month despite describing essentially no countable net income. Zero net income yields the full monthly maximum, annualized to $3,596.04."
-us,scenario_080,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly applied categorical eligibility and the utility-driven shelter deduction but froze the maximum at the FY2025 figure of $292 per month. The benchmark year uses monthly maxima of $298 to $304.68, yielding $3,596.04 annually."
-us,scenario_080,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model stopped at the ordinary $4,250 disabled-household resource limit. Pennsylvania TANF non-cash categorical eligibility supersedes that test here, and the household's assets remain within Pennsylvania's categorical limits."
-us,scenario_080,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model explicitly omitted shelter deductions even though the $2,000 energy subsidy triggers the standard utility allowance. That allowance creates an excess-shelter deduction, reduces net income to zero, and produces the full allotment rather than $239 per month."
-us,scenario_080,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly reached zero net income but used an approximate $291 monthly maximum. The applicable monthly maxima are $298 to $304.68 and annualize to $3,596.04."
-us,scenario_080,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer ignores Pennsylvania TANF non-cash categorical eligibility and the household's very low income. The energy-subsidy shelter deduction eliminates the benefit reduction, so eligibility produces the full allotment."
-us,scenario_080,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified the standard utility allowance and zero net income but used $292 as a flat monthly maximum. Annualizing the applicable $298-to-$304.68 maxima yields $3,596.04."
-us,scenario_080,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model correctly reached categorical eligibility and zero net income but used an outdated $292 monthly maximum. The benchmark's 2026 monthly maxima range from $298 to $304.68 and annualize to $3,596.04."
-us,scenario_080,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly applied categorical eligibility and the heating utility allowance but used $292 per month instead of the applicable 2026 amounts. The changing monthly maxima annualize to $3,596.04."
-us,scenario_080,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer equals a flat $298 monthly allotment for all 12 months. It failed to apply the within-year increase to $304.68, so the correct annualization is $3,596.04."
-us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required structured-output contract."
-us,scenario_080,snap,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The prompt explicitly establishes a one-person household and directs all unlisted numeric inputs to be zero, so the claimed lack of composition and income information is false. Applying those facts gives categorical eligibility and the full annual allotment."
-us,scenario_080,snap,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The prompt identifies the head as the sole listed household member and supplies the relevant assistance and investment income while setting unlisted amounts to zero. SNAP does not require earned income, and this one-person household receives the full allotment under categorical eligibility."
-us,scenario_080,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model explicitly assumed no shelter deduction and therefore reduced the maximum by 30% of $96. The energy subsidy triggers the standard utility allowance and an excess-shelter deduction, reducing net income to zero and eliminating that reduction."
-us,scenario_080,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly omitted a shelter deduction despite the listed energy subsidy. The resulting standard utility allowance reduces net income to zero, so the household receives the full $3,596.04 rather than an income-reduced allotment."
-us,scenario_080,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model stopped after the standard deduction and calculated an income-reduced $270 monthly allotment. It omitted the utility-based excess-shelter deduction triggered by the energy subsidy, which reduces net income to zero."
-us,scenario_080,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the ordinary resource ceiling to the $19,828 bank balance. Pennsylvania TANF non-cash categorical eligibility supplies a different eligibility pathway under which the household's assets pass."
-us,scenario_080,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model stated that no utility expense was present, overlooking the listed $2,000 energy subsidy. That input triggers the standard utility allowance and excess-shelter deduction; absence of rent does not make the benefit zero."
-us,scenario_080,snap,grok-4.5,llm_error,thresholds_rates,False,"The model correctly reached categorical eligibility and zero net income but used the outdated flat maximum of $292 per month. The applicable monthly maxima of $298 to $304.68 annualize to $3,596.04."
-us,scenario_080,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model imposed the ordinary $4,250 disabled-household asset limit. Pennsylvania TANF non-cash categorical eligibility applies instead, and the household's assets pass Pennsylvania's categorical limits."
-us,scenario_080,snap,inkling,llm_error,categorical_eligibility,False,The model incorrectly used the ordinary resource limit rather than Pennsylvania's TANF non-cash categorical-eligibility pathway. The absence of school-age children is irrelevant to SNAP eligibility.
-us,scenario_080,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated the federal disabled-household resource limit as controlling. Pennsylvania TANF non-cash categorical eligibility permits this household's $19,827.65 of assets under the state's applicable limits."
-us,scenario_080,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly found categorical eligibility and zero net income but used a flat $292 monthly maximum. The applicable monthly maximum changes from $298 to $304.68, yielding $3,596.04 for the year."
-us,scenario_080,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model treated the assets and absence of earnings as disqualifying without applying Pennsylvania TANF non-cash categorical eligibility. Earned income is not required, and the household passes the categorical asset and income rules."
-us,scenario_080,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model imposed a $2,750 federal resource limit and incorrectly distinguished a non-elderly disabled person from the categorical pathway. Pennsylvania TANF non-cash categorical eligibility controls and the stated assets pass its limits."
-us,scenario_080,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the household's resources as exceeding the controlling limit and denied eligibility. Pennsylvania TANF non-cash categorical eligibility applies, while the low income and utility-based shelter deduction yield the full allotment."
+us,scenario_080,snap,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly reached zero net income and the full one-person allotment but used a stale $293 monthly maximum. PolicyEngine applies the 2026 monthly maxima of $298 to $304.68 across the year, producing $3,596.04."
+us,scenario_080,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model imposed the ordinary $2,750 resource limit and even added financial assistance to resources. Pennsylvania TANF non-cash categorical eligibility governs this household, under which its $19,827.65 of assets passes the applicable limits."
+us,scenario_080,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached zero net income and a maximum allotment but substituted a $293 monthly estimate. The applicable monthly maximum changes from $298 to $304.68 during the year and annualizes to $3,596.04."
+us,scenario_080,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model applied the ordinary elderly-or-disabled resource ceiling to the bank balance. It omitted Pennsylvania's TANF non-cash categorical-eligibility pathway, under which the household passes the applicable asset rules."
+us,scenario_080,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model replaced the actual benefit computation with an unsupported $218-to-$300 monthly range and selected $2,622. With deductions eliminating the benefit reduction, the household receives the full applicable monthly maxima of $298 to $304.68, totaling $3,596.04."
+us,scenario_080,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,The model correctly applied categorical eligibility and the utility-based excess-shelter deduction to reach zero net income. It then used the FY2025 $292 maximum instead of the 2026 monthly maxima of $298 to $304.68.
+us,scenario_080,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model stopped at the ordinary $4,250 disabled-household resource limit. Pennsylvania's TANF non-cash categorical eligibility supplies a separate qualification pathway under which the household's assets are below the applicable limits."
+us,scenario_080,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model explicitly allowed no shelter deduction despite the $2,000 energy subsidy. That subsidy triggers the standard utility allowance and an excess-shelter deduction, reducing net SNAP income to zero and eliminating the model's 30% income reduction."
+us,scenario_080,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly reduced net income to zero but used a $291 monthly maximum. The applicable monthly maxima are $298 to $304.68 over the year, yielding $3,596.04."
+us,scenario_080,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer treats the assets as disqualifying and ignores Pennsylvania TANF non-cash categorical eligibility. The household passes that pathway, and its low net income yields the full one-person allotment."
+us,scenario_080,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified the utility allowance, zero net income, and maximum allotment. It used a flat $292 monthly maximum instead of the applicable $298-to-$304.68 monthly amounts."
+us,scenario_080,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,The model correctly found categorical eligibility and zero net income after the utility-based shelter deduction. It then applied a stale $292 monthly maximum rather than the 2026 maxima of $298 to $304.68.
+us,scenario_080,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly reached the full one-person allotment through categorical eligibility and the heating utility allowance. Its $3,504 result uses a stale flat maximum instead of the monthly 2026 maxima that annualize to $3,596.04."
+us,scenario_080,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer implies a flat $298 monthly maximum for all 12 months. PolicyEngine applies $298 in part of the year and $304.68 after the applicable update, producing $3,596.04 rather than $3,576."
+us,scenario_080,snap,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SNAP output or explanation, so it failed the required output contract."
+us,scenario_080,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The prompt fixes a one-person household and supplies every unlisted input as zero, so no additional composition or income facts are required. The model failed to apply Pennsylvania categorical eligibility and the SNAP calculation to the stated facts."
+us,scenario_080,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"Earned income is not required for SNAP, and the prompt explicitly establishes a one-person household with unspecified inputs equal to zero. The model incorrectly treated those facts as missing instead of applying Pennsylvania categorical eligibility and the full-allotment calculation."
+us,scenario_080,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model assumed no shelter deduction and therefore retained $96 of monthly net income. The energy subsidy triggers the standard utility allowance and excess-shelter deduction, reducing net income to zero and removing the 30% reduction."
+us,scenario_080,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly omitted a shelter deduction. The energy subsidy generates the standard utility allowance and excess-shelter deduction, which reduces net income to zero and yields the full one-person allotment."
+us,scenario_080,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model stopped after the standard deduction and reduced the allotment for remaining income. It omitted the utility allowance and excess-shelter deduction triggered by the energy subsidy, which reduce net income to zero."
+us,scenario_080,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the ordinary disabled-household resource limit to the $19,828 bank account. Pennsylvania TANF non-cash categorical eligibility permits the household to pass the applicable asset rules."
+us,scenario_080,snap,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model said no utility expense was supplied, overlooking the explicit $2,000 energy subsidy. That input triggers the standard utility allowance and excess-shelter deduction; moreover, absence of shelter costs does not itself make an otherwise eligible household's SNAP allotment zero."
+us,scenario_080,snap,grok-4.5,llm_error,thresholds_rates,False,The model correctly applied categorical eligibility and the utility-based excess-shelter deduction to reach zero net income. It used a stale $292 monthly maximum rather than the applicable $298-to-$304.68 monthly maxima.
+us,scenario_080,snap,grok-4.6,llm_error,thresholds_rates,False,"The model correctly found zero net income and entitlement to the maximum allotment. Its calculation used the older $292 monthly maximum instead of the 2026 monthly amounts that total $3,596.04."
+us,scenario_080,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model imposed the ordinary $4,250 resource ceiling for a disabled household. It omitted Pennsylvania's TANF non-cash categorical-eligibility pathway, under which the stated assets pass."
+us,scenario_080,snap,inkling,llm_error,categorical_eligibility,False,The model wrongly subjected the bank and vehicle assets to the ordinary disabled-household resource ceiling instead of Pennsylvania TANF non-cash categorical eligibility. The absence of school-age children is irrelevant to SNAP eligibility.
+us,scenario_080,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model applied the federal disabled-household resource limit as the sole eligibility route. Pennsylvania TANF non-cash categorical eligibility supplies the controlling pathway and allows the household to pass the applicable asset limits.
+us,scenario_080,snap,kimi-k3,llm_error,thresholds_rates,False,"The model correctly reached zero net income and the maximum one-person benefit but used $292 per month. The 2026 monthly maxima are $298 to $304.68 and annualize to $3,596.04."
+us,scenario_080,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model treated lack of earnings and the asset balance as grounds for ineligibility without performing the categorical-eligibility test. Pennsylvania TANF non-cash categorical eligibility covers the household, and SNAP does not require earned income."
+us,scenario_080,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model applied the ordinary higher resource limit for households with a disabled member. It omitted Pennsylvania TANF non-cash categorical eligibility, through which the household's assets remain within the applicable limits."
+us,scenario_080,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,The model treated the ordinary federal resource ceiling as dispositive and also selected the non-disabled limit despite the stated disability. Pennsylvania TANF non-cash categorical eligibility provides the actual qualification pathway and the household passes its asset rules.
+us,scenario_080,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model inferred ineligibility from assets and no earned income. Pennsylvania TANF non-cash categorical eligibility allows the stated assets, and SNAP eligibility does not require earned income."
 us,scenario_080,ssi,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no SSI output or explanation, violating the required structured-output contract."
 us,scenario_080,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the prompt's general disability fact as establishing SSI disability, despite the applicable engine input is_ssi_aged_blind_disabled=False. Because the 41-year-old is neither SSI-aged, blind, nor SSI-disabled, the federal base-rate and unearned-income calculation does not apply and SSI is $0."
-us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly calculated 100% Pennsylvania Tax Forgiveness and explicitly derived a $0 post-credit liability, then contradicted that derivation by submitting the uncredited $28.86 gross tax. Tax Forgiveness is nonrefundable and therefore reduces state income tax before refundable credits to $0."
-us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,The model stopped after applying Pennsylvania’s 3.07% rate to $940 and omitted the household’s 100% Tax Forgiveness credit. That nonrefundable credit fully offsets the $28.86 gross liability in the requested before-refundable-credits output.
-us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,The model computed only Pennsylvania’s gross tax on the capital gains and dividends and failed to apply 100% Tax Forgiveness. The forgiveness credit is nonrefundable and reduces the requested liability from the gross tax to $0.
-us,scenario_080,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model asserted that no deductions applied and returned the rounded gross tax, but it omitted Pennsylvania’s 100% Tax Forgiveness credit. That nonrefundable credit eliminates the liability before refundable credits; rounding $28.86 to $29 is also inconsistent with the requested amount calculation."
-us,scenario_080,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required submission contract."
-us,scenario_080,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model applied the 3.07% rate to $940 and returned the resulting gross tax without applying Pennsylvania Tax Forgiveness. The household receives 100% forgiveness, which fully offsets that gross liability as a nonrefundable credit."
-us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model correctly excluded the $3,600 financial assistance from taxable income but incorrectly stated that no nonrefundable credits applied. Pennsylvania’s 100% Tax Forgiveness credit fully offsets the $28.86 gross tax, leaving $0 before refundable credits."
-us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
+us,scenario_080,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly derived 100% Pennsylvania Tax Forgiveness and explicitly calculated that it reduces the $28.86 preliminary tax to $0, but then submitted the unreduced $28.86. It failed to carry its own nonrefundable-credit calculation into the output."
+us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,The model stopped after applying Pennsylvania's 3.07% rate to the $940 of dividends and gains. It omitted the 100% nonrefundable Tax Forgiveness credit that eliminates the resulting preliminary liability.
+us,scenario_080,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model computed only gross Pennsylvania tax on the $940 taxable-income base and did not apply Tax Forgiveness. At this household's eligibility income, the 100% nonrefundable forgiveness credit reduces that gross tax to $0."
+us,scenario_080,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated the 3.07% tax on dividends and gains as the final liability and asserted that no deductions applied, but the decisive adjustment is a credit rather than a deduction. Pennsylvania's 100% nonrefundable Tax Forgiveness credit eliminates the preliminary tax before refundable credits."
+us,scenario_080,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested variable, violating the required output contract."
+us,scenario_080,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model submitted the preliminary 3.07% tax on $940 without applying Pennsylvania Tax Forgiveness. The household qualifies for 100% nonrefundable forgiveness, which reduces the preliminary $28.858 liability to $0."
+us,scenario_080,state_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model correctly identified the taxable dividends and capital gains but stopped at gross tax, treating the absence of a standard deduction as dispositive. It omitted the 100% nonrefundable Pennsylvania Tax Forgiveness credit that reduces the liability to $0."
+us,scenario_080,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"After correctly excluding the $3,600 financial assistance from taxable income, the model applied 3.07% to the remaining $940 and incorrectly stated that no nonrefundable credits applied. Pennsylvania Tax Forgiveness applies at 100% for this household and eliminates the preliminary tax."
+us,scenario_080,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable output for state_refundable_credits, violating the required submission contract."
+us,scenario_080,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model misclassified Pennsylvania Schedule SP tax forgiveness as a refundable credit that pays out the $28.86 liability. Schedule SP is nonrefundable and only reduces state income tax, so it contributes $0 to state_refundable_credits."
 us,scenario_080,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included only $5,643 of real-estate tax in SALT and omitted deductible Massachusetts income tax, reducing itemized deductions from $31,028.65 to $22,543. It consequently taxed about $151,550 instead of $143,059.47 and then compounded the error with an unsupported bracket adjustment."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted unreimbursed employee business expenses above the line, used an outdated $14,600 standard deduction, and failed to itemize the $31,028.65 of mortgage-interest and SALT deductions. It also inconsistently discussed subtracting employer premiums and rounded $30,956.92 to the unrelated submitted value of $29,825."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model imposed a $10,000 SALT cap instead of deducting $14,127.41, leaving taxable income too high at about $147,187. Its own bracket computation produced $27,887, but it replaced that result with $31,369 without a valid tax computation."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model capped SALT at $10,000 rather than using the $14,127.41 deduction, overstating taxable income by about $4,128. Its stated bracket arithmetic produced about $28,172, after which it submitted the unsupported and arithmetically inconsistent $31,379."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted deductible Massachusetts income tax from SALT and used only the $5,643 real-estate tax, producing taxable income near $151,544 instead of $143,059.47. It then selected $31,650 from a broad estimate even though its displayed bracket sum was about $28,968."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated real-estate tax as the entire SALT deduction and omitted $8,484.41 of deductible Massachusetts income tax. That omission reduced itemized deductions to $22,543 and raised taxable income to about $151,551, leading directly to the excessive tax."
-us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model capped SALT at $10,000 rather than deducting $14,127.41, so its taxable income was already too high. More decisively, $33,700 does not follow from its stated taxable income of roughly $147,193 under the brackets it claimed to apply."
-us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model improperly subtracted the separately listed employer-sponsored insurance premium from wages and added a personal exemption. It also omitted Massachusetts income tax from itemized SALT and applied a pre-TCJA rate schedule rather than the operative 2026 schedule.
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted a $5,050 personal exemption from taxable income. It also used a restored 10/15/25/28 percent schedule instead of the operative 2026 brackets, so its $29,982.35 does not follow from the correct $143,059.47 taxable income."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used a guessed $15,000 standard deduction and ignored the larger $31,028.65 itemized deduction comprising mortgage interest and deductible SALT. This overstated taxable income and therefore the tax liability."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied TCJA-expiration brackets and a personal exemption that do not govern this 2026 calculation. The correct computation uses $31,028.65 of itemized deductions, a $1.24 QBI deduction, and the operative 2026 single-filer rates."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer insurance premium as a reduction from stated gross wages, improperly added a personal exemption, and omitted Massachusetts income tax from SALT. It then applied an expired-law rate structure rather than the operative 2026 schedule."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly reduced the stated wages by $7,389 of employer-sponsored insurance premiums and included only property tax in SALT. The correct AGI is $174,088.12 and the correct itemized deduction is $31,028.65, producing taxable income of $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The submitted $24,907 does not result from taxing the trace-derived $143,059.47 of taxable income under the 2026 single-filer schedule. The answer implies an unstated excessive deduction or rate reduction beyond the $31,028.65 itemized deduction and $1.24 QBI deduction."
-us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and the pre-TCJA 10/15/25/28 percent brackets. Although it approximated the uncapped SALT deduction, the operative calculation has no personal exemption and taxes $143,059.47 under the 2026 schedule."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model imposed a $10,000 SALT cap instead of using the $14,127.41 deduction, yet still returned a tax far below the amount generated by its resulting taxable income. Its $22,450 is inconsistent with both the correct taxable income of $143,059.47 and the 2026 single-filer brackets."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's $41,495 cannot result from the stated income after either the standard deduction or the available itemized deductions under the 2026 single-filer schedule. The answer implies that deductions or lower bracket layers were not applied correctly."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted an estimated personal exemption in addition to itemized deductions. It also failed to use the trace amounts—AGI of $174,088.12, itemized deductions of $31,028.65, and a $1.24 QBI deduction—before applying the operative 2026 rates."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the stated wages by the separately listed employer-sponsored insurance premium. Gross wages are already the annual wage input, so AGI remains $174,088.12 before the $31,028.65 itemized deduction and $1.24 QBI deduction."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the correct deduction categories but used estimated rather than trace-exact Massachusetts tax and 2026 bracket parameters. The exact inputs produce $14,127.41 of SALT, $143,059.47 of taxable income, and $26,932.27 of tax, not $26,992.59."
-us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model followed the correct itemization pathway but did not apply the exact SALT amount and 2026 bracket parameters. Taxing the trace-derived $143,059.47 yields $26,932.27 rather than $26,966.21."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $151,500 because it omitted deductible Massachusetts income tax from SALT. Including total SALT of $14,127.41 raises itemized deductions to $31,028.65 and lowers taxable income to $143,059.47."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and pre-TCJA 10/15/25/28 percent brackets. The operative 2026 computation has no personal exemption and applies the current schedule to $143,059.47 of taxable income."
-us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted deductible Massachusetts income tax from SALT, leaving taxable income at $151,551 rather than $143,059.47. It also applied a reverted pre-TCJA 10/15/25/28 percent schedule instead of the operative 2026 brackets."
-us,scenario_081,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model approximated Massachusetts income tax and taxable income rather than using the exact trace amounts. Total SALT is $14,127.41, taxable income is $143,059.47, and the exact 2026 bracket calculation produces $26,932.27 rather than $27,056."
-us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the failure occurred before any substantive tax computation could be evaluated."
-us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model counted only the $5,643 real-estate tax in SALT and omitted $8,484.41 of deductible Massachusetts income tax. This reduced itemized deductions to $22,543 and overstated taxable income by the omitted state-tax deduction."
-us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model set the tax to zero despite $143,059.47 of taxable income and no nonrefundable credits. Applying the 2026 single-filer brackets to that taxable income produces a positive $26,932.27 liability."
-us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model falsely stated that $22,543 of itemized deductions was below a $15,000 standard deduction and therefore chose the standard deduction. It also omitted deductible Massachusetts income tax, ignored the full $31,028.65 itemized deduction, and submitted $39,827.88 even though its own displayed bracket sum was $31,281.32."
-us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly chose the standard deduction while simultaneously stating that $22,543 of itemized deductions was lower, even though $22,543 exceeds its assumed standard deduction. More importantly, it omitted deductible Massachusetts income tax that raises total itemized deductions to $31,028.65 and lowers taxable income to $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model deducted only the $5,643 property tax and omitted deductible Massachusetts income tax, reducing itemized deductions to $22,543 instead of $31,028.65. It therefore taxed roughly $8,491 too much income and then used approximate bracket thresholds."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted unreimbursed employee business expenses above the line, treated the listed employer premium inconsistently, and selected a standard deduction even though $31,028.65 of itemized deductions is larger. Its stated $30,956.92 calculation also does not reconcile to the submitted $29,825."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model imposed a $10,000 SALT cap instead of deducting $14,127.41 of state and local taxes under the operative 2026 rules. Its own bracket arithmetic gives $27,887, yet it replaced that result with an unsupported $31,369 estimate."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model capped SALT at $10,000 rather than deducting $14,127.41, leaving taxable income $4,127.41 too high. It then discarded its own computed tax of about $28,172 and submitted the unsupported value $31,379."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only real-estate tax in SALT and omitted deductible Massachusetts income tax, using $22,543 rather than $31,028.65 of itemized deductions. It also submitted $31,650 despite its displayed bracket calculation producing about $28,968."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted deductible Massachusetts income tax from SALT and itemized only $22,543 instead of $31,028.65. That omission raised its taxable income from $143,059.47 to about $151,551 and drove the excess tax."
+us,scenario_081,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model limited SALT to $10,000 instead of the trace’s $14,127.41 deduction and omitted the precise $1.24 QBI deduction. More decisively, $33,700 does not follow from its stated $147,194 taxable income under the bracket schedule it invoked."
+us,scenario_081,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly treated the employer-sponsored premium as a reduction from the reported annual wages, restored a personal exemption, and applied pre-TCJA 10/15/25/28 percent brackets. The operative computation instead uses AGI of $174,088.12, no personal exemption, $31,028.65 of itemized deductions, and the 2026 rate schedule."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,050 personal exemption and used pre-TCJA 10/15/25/28 percent rates. Although it recognized a substantial state-income-tax deduction, those obsolete provisions displaced the operative 2026 computation on $143,059.47 of taxable income."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used an estimated $15,000 standard deduction and ignored the larger $31,028.65 itemized deduction. That shortcut omitted mortgage interest and deductible Massachusetts and real-estate taxes from the taxable-income calculation."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly used TCJA-expired brackets and a restored personal exemption. The operative 2026 calculation has no personal exemption and taxes $143,059.47 under the continuing 10/12/22/24 percent schedule."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model improperly reduced reported wages by the employer premium, restored a $5,300 personal exemption, and applied expired-law rates. It also omitted deductible Massachusetts income tax, limiting itemized deductions to mortgage interest and property tax."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the employer-sponsored premium from the stated wages and omitted deductible Massachusetts income tax from itemized deductions. Those offsetting errors produced $144,151 rather than the traced $143,059.47 of taxable income."
+us,scenario_081,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The unexplained $24,907 is below the $26,932.27 tax generated by $143,059.47 of taxable income under the 2026 single schedule. Its generic reference to itemized deductions supplies no valid computation supporting the additional $2,025.27 reduction."
+us,scenario_081,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, restored a $5,150 personal exemption, and used 10/15/25/28 percent brackets. The operative calculation uses no personal exemption and the 2026 10/12/22/24 percent schedule after $31,028.65 of deductions."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model’s $22,450 is inconsistent with applying the 2026 single-filer brackets to the traced $143,059.47 taxable income. Its terse explanation identifies no deduction or credit capable of reducing the resulting $26,932.27 liability to that amount."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model’s $41,495 does not follow from either the standard-deduction or itemized-deduction calculation described in its explanation; the trace shows those alternatives produce $30,514.85 and $26,932.27. It therefore materially overstated the tax after failing to execute the stated bracket computation."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly added a personal exemption to a 2026 calculation and then used a rate schedule that produces $29,417.08 despite itemized deductions close to the traced amount. No personal exemption applies, and the operative brackets yield $26,932.27 on $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the reported wages by the separately listed employer-sponsored premium. Wages enter AGI at $175,002; after the capital loss and investment income, the traced AGI is $174,088.12 before itemized and QBI deductions."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the correct itemization pathway but used estimated rather than exact 2026 parameters. Applying the operative deductions and brackets to $143,059.47 yields $26,932.27, not $26,992.59."
+us,scenario_081,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified mortgage-interest and SALT itemization but did not execute the exact 2026 bracket calculation. The traced taxable income of $143,059.47 produces $26,932.27, so its $26,966.21 reflects incorrect parameter values or arithmetic."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $151,500, consistent with deducting only mortgage interest and property tax. It omitted deductible Massachusetts income tax, which raises total itemized deductions to $31,028.65 and lowers taxable income to $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA provisions expired, restored a personal exemption, and applied inflation-adjusted 10/15/25/28 percent brackets. The operative 2026 computation uses no personal exemption and the continuing 10/12/22/24 percent schedule."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly restored a $5,300 personal exemption and pre-TCJA 10/15/25/28 percent brackets. Those provisions do not govern this 2026 calculation, which applies the continuing schedule to $143,059.47."
+us,scenario_081,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model omitted Massachusetts income tax from itemized deductions and applied reverted 10/15/25/28 percent brackets. The correct path deducts $14,127.41 of total SALT and uses the operative 2026 10/12/22/24 percent schedule."
+us,scenario_081,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model closely identified the itemized-deduction pathway but estimated the Massachusetts tax deduction and bracket parameters. The exact inputs produce $31,028.65 of total deductions, $143,059.47 of taxable income, and $26,932.27 of tax rather than $27,056."
+us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured result was missing."
+us,scenario_081,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model itemized only mortgage interest and real-estate tax, omitting deductible Massachusetts income tax. This reduced deductions to $22,543 rather than $31,028.65 and overstated taxable income by roughly $8,484."
+us,scenario_081,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model returned zero despite $143,059.47 of taxable income and supplied no credit or other offset. Applying the 2026 single-filer rates produces $26,932.27 before refundable credits."
+us,scenario_081,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model followed the correct itemization structure but estimated Massachusetts income tax, AGI, deduction totals, and bracket thresholds instead of using the exact parameters. The trace uses $31,028.65 of total deductions and $143,059.47 of taxable income, yielding $26,932.27 rather than $26,921."
+us,scenario_081,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model falsely said $22,543 of itemized deductions was lower than a $15,000 standard deduction and therefore selected the standard deduction; even its restricted itemized total is plainly larger. It also submitted $39,827.88 after explicitly computing $31,281.32, so the final number contradicts its own arithmetic."
+us,scenario_081,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction on the false premise that $22,543 of itemized deductions was smaller, while $22,543 already exceeds its implied standard deduction. It also omitted deductible Massachusetts income tax, which brings itemized deductions to $31,028.65 and taxable income to $143,059.47."
 us,scenario_081,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model computed all three required components correctly, including the $805.01 Massachusetts PFML contribution, but then excluded PFML and submitted a number that does not equal even its stated federal subtotal of $13,387.65. The required total is $10,850.12 + $2,537.53 + $805.01 = $14,192.66."
-us,scenario_081,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model used an incorrect $168,600 Social Security wage cap for 2026 and omitted the mandatory $805.01 Massachusetts PFML contribution. It also submitted $13,383.41 despite its own capped-base calculation yielding $12,990.73."
-us,scenario_081,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model omitted the $805.01 Massachusetts PFML employee contribution and then submitted $12,321.73 even though its own Social Security and Medicare calculation totaled $13,387.65. Adding PFML produces $14,192.66."
-us,scenario_081,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model treated the Massachusetts PFML contribution as zero and also submitted $12,717.62 instead of its own correctly stated federal FICA subtotal of $13,387.65. The omitted PFML contribution is $805.01, making the total $14,192.66."
-us,scenario_081,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model treated Massachusetts PFML as approximately zero, omitting the mandatory $805.01 employee contribution. Its submitted $13,385 also fails to match its approximately $13,387.65 Social Security-plus-Medicare computation."
-us,scenario_081,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model rejected the applicable 0.46% Massachusetts PFML employee rate after explicitly computing the correct $805.01 contribution and instead used 0.604%, producing $1,057.01. This overstated PFML by $252 and raised the total from $14,192.66 to $14,444.66."
-us,scenario_081,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly excluded Massachusetts PFML as employer-side, omitting the $805.01 mandatory employee contribution. It then submitted $12,318.65 despite explicitly calculating federal Social Security and Medicare taxes of $13,387.65."
-us,scenario_081,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model applied a 0.38% Massachusetts PFML employee rate instead of 0.46%. That understated the state contribution by about $140 and produced $14,053 rather than $14,192.66."
-us,scenario_081,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied a 0.318% Massachusetts PFML rate and calculated only $556.51 of state payroll tax. The applicable 0.46% contribution is $805.01, so total payroll tax is $14,192.66."
-us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model omitted the $805.01 Massachusetts PFML contribution and its $11,843 submission does not equal 7.65% of the stated $175,002 wage base. Social Security and Medicare alone equal $13,387.65, and PFML raises the total to $14,192.66."
-us,scenario_081,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced payroll-tax wages from $175,002 to $167,613 by subtracting the listed employer-sponsored insurance premiums. PolicyEngine applies Social Security, Medicare, and Massachusetts PFML to the full $175,002 base here, yielding $14,192.66."
-us,scenario_081,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model improperly reduced FICA and PFML wages to $167,613 and also used an understated 0.24% Massachusetts PFML rate. The trace applies the full $175,002 wage base and a $805.01 PFML contribution."
-us,scenario_081,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted the $7,389 employer-sponsored insurance premium from FICA wages even though the traced taxable wage base is the full $175,002. It also omitted the $805.01 Massachusetts PFML contribution."
-us,scenario_081,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The submitted $13,444 includes only about $56 beyond the $13,387.65 federal FICA subtotal, rather than the required $805.01 Massachusetts PFML contribution. Correctly adding PFML yields $14,192.66."
-us,scenario_081,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,The model applied a 0.532% Massachusetts PFML employee rate and calculated approximately $931 instead of the traced 0.46% contribution of $805.01. This overstated total payroll tax by about $126.
-us,scenario_081,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model omitted the mandatory $805.01 Massachusetts PFML employee contribution and submitted a slightly miscomputed federal FICA amount. Social Security and Medicare equal $13,387.65 before PFML, for a final total of $14,192.66."
-us,scenario_081,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model falsely stated that $175,002 exceeds the $200,000 Additional Medicare Tax threshold and produced a number incompatible with the required payroll-tax components. Social Security is $10,850.12, regular Medicare is $2,537.53, Additional Medicare Tax is zero, and Massachusetts PFML is $805.01."
-us,scenario_081,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model correctly calculated federal FICA at $13,387.65 but estimated Massachusetts PFML withholding as $962.51. The traced PFML contribution is $805.01, so the total is $14,192.66."
-us,scenario_081,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,The model used the combined employer-and-employee Medicare rate of 2.9% instead of the employee rate of 1.45%. It also failed to separately apply the $805.01 Massachusetts PFML employee contribution.
-us,scenario_081,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model used a 0.318% Massachusetts PFML employee rate, producing only about $556. The applicable contribution is 0.46%, or $805.01, and total payroll tax is $14,192.66."
-us,scenario_081,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model asserted that Massachusetts has no state payroll tax and therefore omitted the mandatory $805.01 PFML employee contribution. Federal Social Security and Medicare total $13,387.65, but the requested output also includes PFML."
-us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so the required output was missing."
-us,scenario_081,payroll_tax,kimi-k3,llm_error,payroll_tax_base,False,"The model applied a 0.256% Massachusetts PFML rate and calculated $448.01. The applicable 0.46% contribution is $805.01, making total payroll tax $14,192.66."
-us,scenario_081,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omitted every employee payroll-tax component on $175,002 of wages. The required components are $10,850.12 of Social Security, $2,537.53 of Medicare, and $805.01 of Massachusetts PFML."
-us,scenario_081,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly said Massachusetts has no mandatory state payroll tax, omitting the $805.01 PFML employee contribution. It also submitted $14,525.24 despite its own stated federal subtotal being $13,387.65; the correct total with PFML is $14,192.66."
-us,scenario_081,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model gave no component arithmetic and submitted $13,922, which is inconsistent with the $13,387.65 federal FICA subtotal and the required $805.01 Massachusetts PFML contribution. With no Additional Medicare Tax below $200,000, the three applicable components total $14,192.66."
+us,scenario_081,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model correctly calculated all three components, including $805.01 of MA PFML, but then discarded PFML and submitted a number that does not equal even its stated $13,387.65 federal FICA subtotal. It therefore both omitted a mandatory state payroll tax and made an unsupported final arithmetic adjustment."
+us,scenario_081,payroll_tax,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an incorrect $168,600 Social Security wage base for 2026 and excluded the $805.01 MA PFML employee contribution. Its submitted $13,383.41 also contradicts its own capped-FICA calculation of $12,990.73."
+us,scenario_081,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model omitted the $805.01 MA PFML employee contribution after treating it as approximately zero. Its $12,321.73 submission also contradicts its correctly stated federal FICA subtotal of $13,387.65."
+us,scenario_081,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model excluded Massachusetts PFML on the ground that no rate was available, even though the mandatory employee contribution is $805.01. Its $12,717.62 submission also fails to equal its stated Social Security and Medicare components, which sum to $13,387.65."
+us,scenario_081,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model treated MA PFML as approximately zero and therefore omitted its $805.01 employee contribution. It also submitted $13,385 instead of its own approximately $13,387.65 federal FICA calculation."
+us,scenario_081,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a 0.604% MA PFML employee rate and calculated $1,057.01, rather than the applicable 0.46% contribution of $805.01. This overstated payroll tax by $252."
+us,scenario_081,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly classified MA PFML as employer-side and omitted the employee's $805.01 mandatory contribution. Its submitted $12,318.65 also contradicts its explicitly calculated $13,387.65 FICA subtotal."
+us,scenario_081,payroll_tax,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 0.38% MA PFML employee rate instead of 0.46%, producing about $665 rather than $805.01. That rate error understated the total by about $140."
+us,scenario_081,payroll_tax,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an estimated 0.318% MA PFML rate and calculated only $556.51. The applicable 0.46% rate produces $805.01, leaving its total $248.50 too low."
+us,scenario_081,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model said it applied 6.2% Social Security and 1.45% Medicare to $175,002, but those taxes alone equal $13,387.65, not $11,843. It also omitted the $805.01 MA PFML employee contribution."
+us,scenario_081,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model reduced taxable wages to $167,613 by subtracting the listed employer-sponsored insurance premiums, although the trace assesses Social Security, Medicare, and MA PFML on the full $175,002. It also failed to apply the correct $805.01 MA PFML amount."
+us,scenario_081,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced the payroll-tax wage base to $167,613 and applied an approximate 0.24% MA PFML rate. The computation uses full wages of $175,002 and a $805.01 PFML contribution."
+us,scenario_081,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model subtracted $7,389 of employer-sponsored insurance premiums from FICA wages without a basis in the supplied inputs, reducing wages to $167,613. It also omitted the mandatory $805.01 MA PFML employee contribution."
+us,scenario_081,payroll_tax,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted total implies that the model did not use the traced $805.01 MA PFML contribution alongside $13,387.65 of federal FICA. Correctly adding those components produces $14,192.66, not $13,444."
+us,scenario_081,payroll_tax,glm-5.2,llm_error,thresholds_rates,False,"The model applied a 0.532% MA PFML employee rate and calculated about $931. The applicable contribution is 0.46% of $175,002, or $805.01, so it overstated the state payroll tax."
+us,scenario_081,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model limited payroll tax to Social Security and Medicare and omitted the $805.01 MA PFML employee contribution. Its $13,389.37 figure also does not equal the correct FICA subtotal of $13,387.65."
+us,scenario_081,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model incorrectly said $175,002 exceeds the $200,000 Additional Medicare Tax threshold and then submitted $5,420, a figure incompatible with the stated FICA calculation. Social Security and regular Medicare alone are $13,387.65, MA PFML adds $805.01, and Additional Medicare Tax is zero."
+us,scenario_081,payroll_tax,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model correctly calculated federal FICA as $13,387.65 but estimated MA PFML at $962.51. The applicable employee contribution is $805.01, so the state component was overstated by $157.50."
+us,scenario_081,payroll_tax,grok-4.3,llm_error,thresholds_rates,False,"The model applied the combined 2.9% Medicare rate instead of the 1.45% employee rate, thereby including the employer share. It also omitted the separately required $805.01 MA PFML employee contribution."
+us,scenario_081,payroll_tax,grok-4.5,llm_error,thresholds_rates,False,"The model applied a 0.318% MA PFML employee rate and calculated only about $556. The applicable 0.46% contribution is $805.01, causing an understatement of about $249."
+us,scenario_081,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model treated payroll tax as federal FICA only and explicitly excluded any state payroll tax. Massachusetts PFML is a mandatory employee payroll tax and adds $805.01 to the $13,387.65 FICA subtotal."
+us,scenario_081,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, so the required value was missing."
+us,scenario_081,payroll_tax,kimi-k3,llm_error,thresholds_rates,False,"The model used a 0.256% MA PFML rate and calculated $448.01. The applicable rate is 0.46%, yielding $805.01 and raising the total to $14,192.66."
+us,scenario_081,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model returned zero despite $175,002 of employee wages. Those wages generate $10,850.12 of Social Security tax, $2,537.53 of Medicare tax, and $805.01 of MA PFML tax."
+us,scenario_081,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"The model incorrectly stated that Massachusetts has no mandatory employee payroll tax modeled here. It therefore omitted the $805.01 MA PFML contribution from its otherwise correct $13,387.65 federal FICA subtotal."
+us,scenario_081,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly excluded Massachusetts PFML and correctly derived a $13,387.65 federal FICA subtotal, then submitted $14,525.24 without any supporting calculation. The required state contribution is $805.01, producing $14,192.66 rather than the unexplained submitted amount."
+us,scenario_081,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model did not identify or correctly add the $805.01 MA PFML employee contribution and gave an unsupported approximation. At $175,002, Additional Medicare Tax is zero, while Social Security and regular Medicare total $13,387.65 before PFML."
 us,scenario_081,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own component calculation produced approximately $8,238, then it replaced that result with $8,380 without any tax component or arithmetic supporting the added amount. The unexplained final-number substitution caused the error."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $3,835 of nonrefundable dependent and property-tax credits for a single taxpayer with no dependents and no qualifying credit established by the facts. It also treated unreimbursed employee expenses as a Massachusetts itemized deduction instead of deriving the traced Part B base of $169,058 before the personal exemption."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted an estimated $11,287 of FICA from Massachusetts Part B income, although the applicable employee Social Security and Medicare deduction is capped rather than equal to all payroll tax paid. It then submitted $8,730 even though its stated taxable base and 5% calculation produced about $7,966."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model taxed nearly all wages, interest, dividends, and REIT income after only the $4,400 exemption, omitting the adjustments that reduce Part B income before exemption to $169,058. It therefore used $170,768 instead of $164,658 as Part B taxable income and overstated the tax."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model stated a taxable base of about $168,768 and a corresponding tax of about $8,438, then submitted $8,620 based on unspecified “small adjustments.” No identified Massachusetts tax component supports that increase, and it failed to use the traced $164,658 Part B taxable base."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model allowed only a $3,000 renter deduction and consequently derived $167,774 of taxable income rather than the traced $164,658 of Part B taxable income. It also combined dividends with ordinary Schedule A income instead of preserving the separate $110.12 Part A component."
-us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated approximately $8,539 from its stated base and then reduced the answer to $8,200 for unidentified part-year adjustments and rounding, despite the prompt specifying full-year constant Massachusetts residence. That unsupported adjustment displaced both its own calculation and the required Part A/Part B computation."
-us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model imported a federal AGI of $166,705 and deducted $22,543 of itemized expenses plus an estimated $5,000 exemption. Massachusetts instead derives Part B income before exemption as $169,058 and uses the $4,400 single exemption, so the model understated the taxable base substantially."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model started from wages alone and subtracted personal, payroll-tax, and rental amounts to obtain $165,602, rather than using the traced Part B taxable income of $164,658 after exemption. It also omitted the separately taxed $110.12 Part A dividend component."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer is exactly 5% of roughly $175,000, showing that the model applied the headline rate to gross income. It omitted the adjustments producing $169,058 of Part B income before exemption, the $4,400 personal exemption, and the separate Part A dividend calculation."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model asserted a $157,219 Part B taxable base after listed deductions, but those deductions from the household's income do not arithmetically produce that figure. The traced Part B base is $169,058 before the $4,400 exemption and $164,658 afterward, with Part A dividends taxed separately."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model substituted “federal taxable wages” of $167,613 for Massachusetts Part B income and treated the capital loss as offsetting interest and dividends. Massachusetts's traced computation instead gives $169,058 before the personal exemption and retains $110.12 of separately taxable Part A dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model began with an unsupported $167,613 wage base and then deducted the personal exemption, renter deduction, and payroll-tax deduction again, driving taxable income down to $157,213. The correct Part B sequence yields $169,058 before exemption and $164,658 after exemption, plus separate Part A dividend tax."
-us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The submitted tax implies only about $157,380 of income taxed at 5%, consistent with excessive deductions for FICA and rent. The traced Part B taxable base is $164,658 after the personal exemption, and the $110.12 Part A dividend component must also be taxed."
-us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model improperly used the $1,080 short-term capital loss to reduce ordinary Massachusetts income and failed to subtract the $4,400 personal exemption. It also folded all income into one 5% base rather than using $164,658 of Part B taxable income and a separate $110.12 Part A dividend component."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $6,440 equals 5% of only $128,800, so the model implicitly removed about $40,000 more income than the Massachusetts computation permits. The correct derivation taxes $164,658 of Part B income after exemption and $110.12 of Part A dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's $13,951 cannot result from applying Massachusetts's ordinary 5% rate to this household's income and is consistent with importing an inapplicable rate or tax structure. Income is below the Massachusetts surtax threshold, and the traced Part A/Part B computation yields tax on $164,658 of Part B income plus $110.12 of dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model erased the entire $166 interest-and-dividend amount with the short-term capital loss, eliminating the separately taxable $110.12 Part A dividend component. Its Part B base of $164,602 also differs from the traced $164,658, so both the base and category treatment were wrong."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The answer implies about $158,300 of income taxed at 5%, showing that the model deducted substantially more than the Massachusetts computation allows. The correct Part B taxable base after the exemption is $164,658, with tax on $110.12 of Part A dividends added separately."
-us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model reduced income using only a personal exemption and a maximum renter deduction but did not reproduce the traced Part B income-before-exemption figure of $169,058. It also omitted the separate treatment of $110.12 in Part A dividends, leading to the wrong total."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied 5% directly to approximately $174,094 of net income. It omitted the adjustments and $4,400 exemption that reduce Part B taxable income to $164,658 and did not calculate the $110.12 Part A dividend component separately."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model fully deducted $1,782 of unreimbursed employee expenses and started from an income figure that netted the short-term capital loss against ordinary income. Those treatments bypassed the traced $169,058 Part B base before exemption and the separate $110.12 Part A dividend tax."
-us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model treated $174,094 as Massachusetts ordinary income and subtracted only the personal exemption. It failed to apply the adjustments producing $169,058 of Part B income before exemption and failed to tax $110.12 of Part A dividends separately."
-us,scenario_081,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used federal AGI, a $3,000 rent deduction, and the personal exemption to estimate a single taxable base of $166,694. Massachusetts's traced calculation instead produces $164,658 of Part B taxable income after exemption and adds tax on $110.12 of Part A dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. This is a missing-output contract failure rather than a substantive tax calculation.
-us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $2,000 payroll-tax deduction and the $4,400 exemption from its $174,088 income base, producing $167,688 instead of the traced $164,658 Part B taxable amount. It also omitted the separate tax on $110.12 of Part A dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model returned zero despite the household being a Massachusetts resident with substantial Massachusetts-taxable wage income. The state computation taxes $164,658 of Part B income after exemption and $110.12 of Part A dividends."
-us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model netted the $1,080 short-term capital loss into ordinary income, subtracted only the personal exemption, and applied one 5% rate to the result. It omitted the adjustments yielding $169,058 of Part B income before exemption and failed to preserve the separately taxable $110.12 Part A dividend amount."
-us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The submitted $7,800 implies a taxable base of only $156,000 at the stated 5% rate, well below the traced $164,658 Part B taxable amount. The model therefore applied unsupported deductions and also omitted the separate Part A dividend tax."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's stated component calculation already produced about $8,238, but it submitted $8,380 after an unexplained $142 increase. It failed to carry its own near-correct calculation into the final value; the traced Part A and Part B taxes total $8,238.41."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $3,835 of property-tax and dependent-related nonrefundable credits even though this single taxpayer has no dependent and no identified applicable credit. It also treated unreimbursed employee expenses as a Massachusetts itemized deduction instead of deriving the traced $169,058 Part B income before exemption."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model deducted an estimated $11,287 of FICA from wages and then abandoned the resulting $7,966 calculation for an unexplained $8,730 submission. Massachusetts Part B income before exemption is $169,058, not the model's FICA-reduced wage base, and the remaining $110.12 of Part A dividends also must be taxed."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model taxed essentially all wages, interest, and dividends after only the personal exemption, yielding a Part B base of $170,768. It missed the adjustments that reduce traced Part B income before exemption to $169,058 and failed to preserve the separate $110.12 Part A component."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model calculated about $8,438 from its stated base and then added unspecified adjustments to submit $8,620. It neither derived the traced $164,658 taxable Part B base nor separately taxed $110.12 of Part A dividend income."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used $175,174 as Massachusetts income and deducted only a $3,000 renter deduction and the personal exemption, producing an overstated $167,774 taxable base. The trace instead gives $169,058 of Part B income before the $4,400 exemption and keeps $110.12 of dividends in Part A."
+us,scenario_081,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's explicit calculation produced about $8,539, but it reduced that to $8,200 using nonexistent 'part-year adjustments and rounding' despite the prompt specifying a full-year Massachusetts resident. It also failed to derive the traced Part A/Part B split."
+us,scenario_081,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model imported a federal AGI of $166,705, invented $22,543 of Massachusetts itemized deductions, and estimated a $5,000 exemption. Massachusetts instead uses $169,058 of Part B income before the fixed $4,400 personal exemption, plus separately taxed Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model constructed Part B income directly from wages less assumed $2,000 payroll-tax and $3,000 rent deductions, producing $165,602 after exemption. The traced taxable Part B amount is $164,658, and $110.12 of Part A dividends adds $5.51 of tax."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer simply applied 5% to roughly $175,000 of adjusted gross income, omitting the Massachusetts adjustments and $4,400 personal exemption. It also failed to separate $110.12 of Part A dividends from the $164,658 taxable Part B base."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $157,219 taxable Part B base despite listing deductions totaling only $10,400. The traced base is $169,058 before the $4,400 exemption, or $164,658 afterward, with an additional tax on $110.12 of Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model substituted an unsupported $167,613 'federal taxable wages' figure for Massachusetts Part B income and treated interest and dividends as fully erased by the capital loss. The trace yields $169,058 before exemption and retains $110.12 as taxable Part A dividend income."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model began from an unsupported $167,613 wage base and then stacked rent, payroll-tax, and personal-exemption deductions to reach $157,213. Massachusetts taxable Part B income is $164,658, and the calculation also includes tax on $110.12 of Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount implies a taxable base near $157,380, reflecting excessive deductions for FICA and rent from the Massachusetts Part B base. The traced taxable Part B base is $164,658, with $110.12 of separately taxed Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly allowed the $1,080 short-term capital loss to offset ordinary income and omitted the $4,400 personal exemption while applying only a rent deduction. It therefore never derived the traced $164,658 taxable Part B income or the surviving $110.12 Part A dividend amount."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The unexplained $6,440 answer implies only $128,800 taxed at 5%, far below the traced Massachusetts bases. The correct computation taxes $164,658 of Part B income and $110.12 of Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The $13,951 answer implies an effective tax far above Massachusetts's applicable 5% rates at this income and invokes unspecified standard or itemized deductions. The 4% millionaires surtax does not apply, and the traced Part A and Part B computation totals $8,238.41."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used $164,602 instead of the traced $164,658 taxable Part B base by constructing deductions directly from wages. It also erased all interest and dividend income with the short-term loss, whereas the trace retains $110.12 of taxable Part A dividends, adding $5.51."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $7,915 estimate implies a taxable base of about $158,300, reflecting excessive or unsupported reductions for the exemption and renter deduction. The traced computation uses $164,658 of taxable Part B income and separately taxes $110.12 of Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model reduced income using its assumed maximum renter deduction but did not derive the traced $169,058 Part B income before exemption. The correct taxable Part B base is $164,658, with $110.12 of Part A dividends taxed separately."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model applied 5% directly to approximately $174,094, omitting the Massachusetts personal exemption and other adjustments embedded in traced Part B income. It also failed to separate Part A dividends from Part B ordinary income."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly treated the full $1,782 of unreimbursed employee expenses as a Massachusetts deduction and used net federal-style income of $174,094 as its starting point. The trace instead establishes $169,058 of Part B income before exemption and $110.12 of separate Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model treated the capital loss as a $1,000 reduction of the combined income base and then deducted $2,000 for FICA, producing $167,774. The trace instead yields $164,658 of taxable Part B income and retains $110.12 of taxable Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model started from a federal-style net income of $174,094 and subtracted only the personal exemption. It missed the adjustments that set Part B income before exemption at $169,058 and failed to preserve $110.12 as separate Part A dividend income."
+us,scenario_081,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model started from $174,094 and mechanically deducted a $3,000 renter deduction and $4,400 exemption, yielding about $166,694. The trace instead gives $164,658 of taxable Part B income plus $110.12 of taxable Part A dividends."
+us,scenario_081,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no value or explanation for the requested output. It therefore failed the required structured-output contract before any Massachusetts tax computation could be evaluated.
+us,scenario_081,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $2,000 payroll-tax deduction and the $4,400 exemption from a combined $174,088 income base. It missed the traced Part A/Part B classification: taxable Part B income is $164,658 and $110.12 remains separately taxable in Part A."
+us,scenario_081,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer omitted the entire Massachusetts liability despite $164,658 of taxable Part B income and $110.12 of taxable Part A dividends. Applying the 5% taxes to those traced components yields $8,238.41."
+us,scenario_081,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model treated wages less only the personal exemption as the Part B base and taxed all $166 of stated interest and dividends separately. The trace reduces Part B income before exemption to $169,058 and identifies only $110.12 as taxable Part A dividend income."
+us,scenario_081,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model netted the short-term capital loss directly against wages and investment income, then subtracted only the personal exemption. It failed to derive the traced $169,058 Part B amount before exemption and the separate $110.12 Part A dividend component."
+us,scenario_081,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The $7,800 estimate implies only about $156,000 of income taxed at 5%, with no derivation for that reduced base. The trace establishes $164,658 of taxable Part B income plus $110.12 of taxable Part A dividends."
 us,scenario_081,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_082,child1_chip_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model incorrectly treated age 1 as within the applicable CHIP age range. The trace applies CHIP's age criterion against the child and returns false, while Medicaid eligibility also returns NONE."
 us,scenario_082,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model omitted the requested child1_chip_eligible output and supplied no explanation, violating the required submission contract."
@@ -4935,36 +5182,38 @@ us,scenario_082,child1_medicaid_eligible,minimax-m3,llm_error,categorical_eligib
 us,scenario_082,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_082,child1_wic_eligible,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model incorrectly asserted that the household met the WIC income requirement. It failed to compare approximately $90,132 of annual household income with the much lower 185%-of-poverty limit for a two-person household."
 us,scenario_082,child1_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the child's age under five as sufficient for WIC eligibility. Age establishes categorical eligibility only; the household must also pass the 185%-of-poverty income test, which it fails at approximately $90,132 of annual income."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model imposed a $3,000 capital-loss shortcut and derived $93,436 of taxable income instead of the traced $92,735.15. Its submitted $14,748 also contradicts its own final recomputation of approximately $10,353, so it did not submit the result supported by its reasoning."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model applied the single-filer standard deduction despite recognizing that the filer has a qualifying child and files head of household. It also used $1,920 of CDCC instead of the one-dependent $3,000 expense cap times 20%, and its $23,437 submission is disconnected from its stated arithmetic."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used estimated head-of-household bracket thresholds and a $23,500 standard deduction rather than the traced 2026 parameters. That raised ordinary tax to about $10,922 instead of $10,349.45; the preferential tax and $2,800 of credits were otherwise identified correctly."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used approximate deductions, brackets, and a $2,000 CTC instead of the traced $24,150 standard deduction, exact rate schedule, and $2,200 CTC. Its submitted $11,150 also does not equal its own stated $10,460 calculation."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model substituted estimated 2026 bracket thresholds and an approximately $24,500 standard deduction for the traced parameters. This produced about $12,730 before credits rather than $12,363.05, even though it correctly subtracted the $600 CDCC and $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,424 of qualified dividends when forming AGI, arriving at $106,519, and then used an estimated $22,500 standard deduction. The correct taxable-income computation is $92,735.15, followed by $12,363.05 of tax and $2,800 of nonrefundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used estimated 2026 deductions and brackets and stated a $2,000 CTC rather than $2,200. Its submitted $13,920 additionally ignores its own final calculation of $10,739."
-us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied obsolete personal exemptions, an unsupported $26,228 itemized deduction, old 10/15/25% brackets, and a $1,000 CTC. The traced calculation instead uses the $24,150 standard deduction, no personal exemptions, current rates, and a $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model doubled the listed real-estate tax into $18,564 of SALT, itemized it, and then also subtracted personal exemptions. The correct deduction is the $24,150 standard deduction with no personal exemptions, and the applicable CTC is $2,200 rather than $1,000."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $100,571 AGI and a $2,000 CTC. The traced computation yields $92,735.15 of taxable income and subtracts a $2,200 CTC plus $600 CDCC from $12,363.05."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted both itemized deductions and personal exemptions, neither of which is the traced deduction treatment. It also mentions only the $600 CDCC and omits the $2,200 nonrefundable CTC from its stated credit calculation."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used personal exemptions in addition to the head-of-household standard deduction and allowed only a $1,000 CTC. The traced calculation uses no personal exemptions and subtracts $2,200 of CTC plus $600 of CDCC."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies excessive deductions because $7,380 is far below the $9,563.05 obtained from $92,735.15 of taxable income. The correct computation produces $12,363.05 before credits and subtracts exactly $2,800 of nonrefundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $3,384 answer implies that the model removed substantially too much income or claimed unsupported deductions or credits. The traced taxable income produces $12,363.05 before credits, and only $2,800 is subtracted before refundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $23,063 standard deduction and estimated bracket cutoffs, producing $94,531 of taxable income and $13,061.38 before credits. The traced values are a $24,150 standard deduction, $92,735.15 of taxable income, and $12,363.05 before credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model's rough treatment of wages, gains, dividends, and the standard deduction did not reproduce $92,735.15 of taxable income. Its $12,719 answer is even above the traced $12,363.05 tax before any credits, so the required $600 CDCC and $2,200 CTC were not correctly reflected."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model used an unsupported itemized-deduction estimate based on real-estate tax and medical expenses rather than the traced $24,150 standard deduction. Its $17,303 result also fails to reflect the preferential treatment of $13,424 and the full $2,800 of nonrefundable credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed about $24,415 of itemized deductions even though the traced computation uses the $24,150 standard deduction and $3,008.48 of above-the-line deductions. That produced the wrong taxable income and $12,497 of pre-credit tax instead of $12,363.05."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used approximately $89,951 as AGI and thereby understated the income subject to tax. The traced computation yields $92,735.15 of taxable income, $12,363.05 before credits, and $9,563.05 after the stated $2,800 of credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the $2,200 CTC and $600 CDCC but used estimated taxable-income and rate parameters. The exact schedule produces $10,349.45 of ordinary tax plus $2,013.60 of preferential tax, leaving $9,563.05 after credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model named the correct credit amounts but overstated the tax generated by the head-of-household brackets. The exact pre-credit tax is $12,363.05, not the $13,036 implied by subtracting $2,800 from its answer."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The rounded $14,000 estimate does not implement the traced computation: $10,349.45 of ordinary tax plus $2,013.60 of preferential tax, less $2,800 of credits. It overstates even the $12,363.05 liability before those credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly excluded the $21,208 employer-sponsored insurance premiums from wages, added post-TCJA personal exemptions, and treated the CTC as fully phased out. The trace instead uses $92,735.15 of taxable income with no personal exemptions and allows the full $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied a post-TCJA-expiration $12,591 standard deduction and omitted the $2,200 CTC. The traced 2026 rules use a $24,150 head-of-household standard deduction and subtract both the $600 CDCC and $2,200 CTC."
-us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model was close on taxable income and credits but used approximate brackets that produced $10,638 of ordinary tax. The exact ordinary-rate computation is $10,349.45, with $2,013.60 of preferential tax and $2,800 of credits."
-us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required numeric answer could not be parsed."
-us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly added a separate $700 non-itemizer charitable deduction on top of the $24,150 standard deduction, producing $92,812 rather than $92,735.15 of taxable income. It consequently computed $12,379.96 before credits instead of $12,363.05."
-us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $30,711 net capital loss as capable of eliminating the remaining taxable income and tax. The traced computation still has $92,735.15 of taxable income and $12,363.05 of tax before the limited $2,800 nonrefundable-credit reduction."
-us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $30,711 net capital loss directly in AGI, used an estimated $21,900 standard deduction, and treated only $400 of the CTC as nonrefundable. The trace instead yields $92,735.15 of taxable income and uses the full $2,200 CTC against available liability; its submitted $12,866.90 also contradicts every calculation in its explanation."
-us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented $139,775 of AGI and a $6,237 QBI deduction despite no qualified business income, used single filing status, and allowed only $1,875 of CTC. The filer is head of household, the traced taxable income is $92,735.15, and the allowed nonrefundable credits total $2,800."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model’s displayed recomputation used estimated head-of-household bracket thresholds and produced ordinary tax inconsistent with the applicable 2026 schedule. It also omitted the $700 charitable deduction and then submitted $14,748 despite its own final calculation of about $10,353."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model applied the single-filer standard deduction even though the filer qualifies as head of household. It also treated all $9,600 of childcare expenses as creditable instead of applying the $3,000 one-dependent cap, and its submitted $23,437 contradicts its stated calculation."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the applicable $700 charitable deduction and used an estimated $23,500 standard deduction rather than $24,150. Those errors raised taxable income and ordinary tax above the trace-derived amounts."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used approximate deductions, brackets, and a $2,000 CTC instead of the applicable $24,150 standard deduction, $700 charitable deduction, exact 2026 brackets, and $2,200 CTC. It then submitted $11,150 despite its own calculation ending at $10,460."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model approximated the standard deduction as $24,500 and omitted the applicable $700 charitable deduction. This produced the wrong taxable-income base and ordinary-rate tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model excluded the $13,424 of qualified dividends from AGI while still taxing them later, reducing taxable income by that amount. It also used estimated deduction and bracket parameters and a $2,000 CTC instead of $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $13,920 does not follow from the model’s own derivation, which ended at $10,739 after credits. That derivation also used an estimated standard deduction, omitted the $700 charitable deduction, and used a $2,000 rather than $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied obsolete personal exemptions, 10/15/25 percent ordinary brackets, and a $1,000 CTC. The applicable computation instead uses the 2026 head-of-household schedule, no personal exemptions, and a $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model invented $18,564 of SALT deductions and personal exemptions, neither of which belongs in the reference computation. It also reduced liability by only a $1,000 CTC instead of $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $100,571 AGI omits or misnets income and deductions relative to the traced gross-income computation. The model also used only a $2,000 CTC rather than the applicable $2,200 credit."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $96,446 AGI, unsupported itemized deductions, and personal exemptions. The correct path uses the $24,150 standard deduction plus the applicable $700 charitable deduction and subtracts both the $600 CDCC and $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $96,454 AGI and personal exemptions, lowering the tax base outside the traced calculation. It also allowed only a $1,000 CTC instead of $2,200."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies a substantially understated taxable-income or tax base. The trace yields $92,735.15 of taxable income, $12,363.05 before credits, and $9,563.05 after the $2,800 of nonrefundable credits."
+us,scenario_082,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer implies that the model removed far more income or credits than allowed. The correct taxable income is $92,735.15 and the only identified nonrefundable credits total $2,800, leaving $9,563.05."
+us,scenario_082,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an estimated $23,063 standard deduction and omitted the applicable $700 charitable deduction. This overstated taxable income relative to $92,735.15 and therefore overstated ordinary tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model’s rough calculation failed to apply the traced deductions and preferential-income computation that produce $92,735.15 of taxable income and $12,363.05 of pre-credit tax. After the $600 CDCC and $2,200 CTC, the result is $9,563.05 rather than $12,719."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model relied on an unsupported itemized-deduction estimate instead of the $24,150 standard deduction and applicable $700 charitable deduction. Its rounded liability also fails to apply the traced preferential-rate tax and full $2,800 of nonrefundable credits."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model described $24,415 as itemized deductions even though the traced computation uses the $24,150 standard deduction and the applicable $700 charitable deduction. That incorrect deduction total led to the wrong taxable income and pre-credit tax."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model understated the income base by using approximately $89,951 of AGI. The traced computation instead produces $92,735.15 of taxable income after the specified deductions, resulting in $12,363.05 before credits."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Although the model identified the main income components and credits, its rounded estimate did not apply the exact 2026 head-of-household bracket calculation. The correct ordinary tax is $10,349.45 and preferential-rate tax is $2,013.60 before the $2,800 credit reduction."
+us,scenario_082,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model’s answer implies that it overstated the head-of-household ordinary-rate tax. Applying the exact brackets to the traced taxable-income split produces $12,363.05 before the stated $2,200 CTC and $600 CDCC."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The unsupported rounded estimate does not implement the traced calculation. Tax on the $92,735.15 taxable-income base is $12,363.05 before the $2,800 of nonrefundable credits, leaving $9,563.05."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded $21,208 of employer-sponsored insurance premiums from the stated wages and then applied personal exemptions. It also treated the CTC as fully phased out under obsolete post-sunset rules instead of allowing the $2,200 credit."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied obsolete post-sunset personal exemptions, 10/15/25 percent brackets, and a $75,000 CTC phaseout threshold. The applicable calculation uses the 2026 head-of-household schedule and the full $2,200 CTC."
+us,scenario_082,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model applied a post-TCJA $12,591 standard deduction instead of the applicable $24,150 head-of-household deduction. It also omitted the $2,200 CTC entirely, subtracting only the $600 CDCC."
+us,scenario_082,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model’s approximate ordinary-bracket calculation produced $10,638 instead of the traced $10,349.45. It also used an approximate $93,504 taxable-income base rather than $92,735.15."
+us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
+us,scenario_082,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model applied the $700 charitable deduction on top of a $24,150 standard deduction but calculated taxable income as $92,812 rather than the traced $92,735.15. Its resulting ordinary tax was $16.91 too high, producing the corresponding overstatement after credits."
+us,scenario_082,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $30,710.82 net capital loss as capable of eliminating the remaining tax base rather than applying the traced capital-loss and deduction computation. Taxable income remains $92,735.15, so the CTC and CDCC cannot reduce liability to zero."
+us,scenario_082,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used an approximate $93,512 taxable-income base and approximate brackets, overstating ordinary tax. The traced taxable income is $92,735.15 and exact ordinary tax is $10,349.45."
+us,scenario_082,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the entire $30,711 net capital loss from current-year income instead of following the traced $3,008.48 above-the-line deduction computation. It also treated only $400 of the CTC as nonrefundable, whereas the full $2,200 credit is usable against liability, and its submitted value contradicts every calculation in its explanation."
+us,scenario_082,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI to $139,775, used single filing status, and invented a $6,237 QBI deduction despite no qualified business income. It also used the wrong CTC amount instead of the full $2,200 credit available to this head-of-household filer."
 us,scenario_082,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the $1,700 ACTC cap as an automatic refundable credit after acknowledging that the filer had sufficient tax liability to use the CTC nonrefundably. Because the full CTC offsets tax, no unused CTC remains for the ACTC, so refundable credits equal zero."
 us,scenario_082,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model assigned a $1,000 ACTC without first determining how much CTC remained after offsetting federal income tax. The household's tax liability absorbs the full CTC, leaving zero refundable ACTC."
 us,scenario_082,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly treated the child credit as fully refundable and used an unsupported $4,000 amount for one child. Under 2026 law the CTC is applied nonrefundably against this household's sufficient tax liability, leaving no refundable ACTC."
@@ -4978,96 +5227,102 @@ us,scenario_082,local_income_tax,gemini-3.1-flash-lite-preview,llm_error,state_l
 us,scenario_082,local_income_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model calculated NYC head-of-household tax brackets on $79,254 of taxable income without any fact establishing NYC residence. New York State residence alone triggers no local income tax in this benchmark, yielding zero."
 us,scenario_082,local_income_tax,glm-5.2,llm_error,state_local_rule,False,"The model explicitly assumed NYC residence because NYC is a New York locality with an income tax, contradicting the instruction that unlisted household facts are false. Its detailed NYC taxable-income, bracket, school-credit, and dependent-care-credit computation is inapplicable because no NYC location was provided."
 us,scenario_082,local_income_tax,inkling,llm_error,state_local_rule,False,"The model treated the household as an NYC resident and applied NYC brackets and credit rules despite the prompt listing only New York State. Without a specified NYC residence, none of those local rules applies and the local-income-tax output is zero."
-us,scenario_082,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model explicitly excluded New York PFL and disability contributions after correctly calculating $7,664.92 of federal FICA. It omitted $411.91 of PFL and $31.20 of disability contributions."
-us,scenario_082,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used an estimated 0.62% New York PFL rate and then inserted unspecified additional state payroll taxes. The required state components are $411.91 of PFL and $31.20 of disability contributions, totaling $443.11 rather than $640.00."
-us,scenario_082,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,The model vacillated among incompatible PFL estimates and ultimately included only $80.51 of New York contributions. It needed to add $411.91 of PFL plus $31.20 of disability contributions to federal FICA.
-us,scenario_082,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model stated that New York has no mandatory employee payroll tax and stopped at federal FICA. It omitted the $411.91 PFL and $31.20 disability contributions.
-us,scenario_082,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model correctly computed federal Social Security and Medicare taxes but incorrectly asserted that New York imposes no employee payroll tax counted here. New York PFL and disability contributions add $443.11.
-us,scenario_082,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly included the $31.20 disability contribution but applied an obsolete estimated PFL rate and wage cap, producing $346.65. The PFL component is $411.91 on the $100,195 wage base."
-us,scenario_082,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model treated New York PFL and disability contributions as unspecified and substituted a combined estimate of only $129.39. These mandatory contributions total $443.11: $411.91 for PFL and $31.20 for disability benefits.
-us,scenario_082,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,The model included the $31.20 New York disability contribution but omitted the $411.91 Paid Family Leave contribution. Its federal FICA computation was otherwise exact.
-us,scenario_082,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used an approximate $333 PFL contribution instead of $411.91 and rounded the disability contribution rather than using $31.20. Those state components must add $443.11 to the exact $7,664.92 federal FICA amount."
-us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model applied only the 7.65% federal FICA rate and omitted both mandatory New York components. PFL contributes $411.91 and disability benefits contribute $31.20.
-us,scenario_082,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted the $21,208 employer-sponsored-insurance premium from wages, reducing the FICA base to $78,987. PolicyEngine applies Social Security, Medicare, PFL, and disability contributions to the full $100,195 of taxable wages, with the state components equal to $411.91 and $31.20."
-us,scenario_082,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly treated the listed employer-sponsored-insurance premiums as a pre-tax reduction of FICA wages and taxed only $78,987. The payroll-tax base is the full $100,195, and the exact New York additions are $411.91 of PFL and $31.20 of disability contributions."
-us,scenario_082,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model reduced taxable wages to $78,987 by subtracting employer-sponsored-insurance premiums. All four employee payroll-tax components are calculated from the stated $100,195 wage base, producing $8,108.03."
-us,scenario_082,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model incorrectly used $78,987 as FICA-taxable wages after deducting employer-sponsored-insurance premiums. The correct wage base is $100,195, and New York PFL and disability contributions add $443.11."
-us,scenario_082,payroll_tax,glm-5.2,llm_error,state_local_rule,False,The model correctly computed federal FICA and the $31.20 disability contribution but used an estimated PFL rate and cap to obtain $373.26. The applicable PFL contribution is $411.91.
-us,scenario_082,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly excluded mandatory New York employee payroll taxes and also rounded federal FICA below its exact $7,664.92 value. It needed to add $411.91 of PFL and $31.20 of disability contributions."
-us,scenario_082,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,The model rounded the federal Social Security and Medicare computation and stopped there. It omitted $411.91 of New York PFL and $31.20 of New York disability contributions.
-us,scenario_082,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"The model correctly obtained $7,664.92 of federal FICA but estimated the combined New York PFL and disability contributions at $385.73. Their exact total is $443.11."
-us,scenario_082,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,The model included the $31.20 New York disability contribution but omitted the $411.91 Paid Family Leave contribution. Adding that component yields the requested total.
-us,scenario_082,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,The model calculated only federal Social Security and Medicare taxes. It omitted New York's $411.91 PFL and $31.20 disability contributions.
-us,scenario_082,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model applied only the federal FICA rates, omitted both New York employee contributions, and misstated even the federal subtotal, which is $7,664.92. The state additions are $411.91 for PFL and $31.20 for disability benefits."
-us,scenario_082,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model used only the 7.65% federal FICA shortcut. That shortcut excludes $443.11 of mandatory New York PFL and disability contributions.
-us,scenario_082,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly subtracted $21,208 of employer-sponsored-insurance premiums from FICA wages and also excluded New York payroll contributions. The applicable wage base is $100,195, and the state components are $411.91 of PFL and $31.20 of disability contributions."
-us,scenario_082,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly deducted both the $21,208 insurance premiums and the $181 traditional 401(k) contribution from FICA wages. Neither deduction reduces the $100,195 payroll-tax base here, and the model also omitted $443.11 of mandatory New York contributions."
-us,scenario_082,payroll_tax,inkling,llm_error,state_local_rule,False,"The model recognized both New York programs but estimated PFL at about $365 instead of $411.91 and rounded all components. The exact state total is $443.11, which combines with $7,664.92 of federal FICA."
-us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax output or explanation, so it failed the required output contract."
-us,scenario_082,payroll_tax,minimax-m3,llm_error,state_local_rule,False,The model correctly approximated federal FICA but omitted New York Paid Family Leave and disability-benefits contributions. Those mandatory state components add $443.11.
-us,scenario_082,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model cycled through contradictory New York contribution estimates and then submitted a total below federal FICA by inserting a negative state adjustment. New York adds $411.91 of PFL and $31.20 of disability contributions; it does not subtract from the $7,664.92 federal subtotal."
-us,scenario_082,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model stated that no mandatory state payroll tax was included, omitting $443.11 of New York PFL and disability contributions. Its submitted $7,752.42 also does not equal the $7,664.92 federal components stated in its own reasoning."
+us,scenario_082,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model expressly excluded New York Paid Family Leave and Disability Benefits contributions. Those mandatory employee payroll taxes add $411.91 and $31.20 to the correctly calculated $7,664.92 of federal FICA."
+us,scenario_082,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model used an incorrect 0.62% New York Paid Family Leave estimate and then inserted unspecified additional state payroll taxes. The required New York components are $411.91 of PFL and $31.20 of Disability Benefits, totaling $443.11 rather than $640.00."
+us,scenario_082,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,The model vacillated among incompatible PFL rates and caps and ultimately included only $80.51 of New York payroll taxes. The correct state addition is $411.91 of PFL plus $31.20 of Disability Benefits.
+us,scenario_082,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly stated that New York has no mandatory employee payroll tax. It omitted $411.91 in Paid Family Leave contributions and $31.20 in Disability Benefits contributions.
+us,scenario_082,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,The model included only federal Social Security and Medicare and incorrectly excluded mandatory New York employee payroll taxes. It omitted $443.11 consisting of $411.91 PFL and $31.20 Disability Benefits.
+us,scenario_082,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied an obsolete 0.388% PFL rate and an incorrect wage cap, producing $346.65 instead of $411.91. Its $31.20 Disability Benefits amount and federal FICA calculation were correct."
+us,scenario_082,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,The model assigned only $129.39 to New York SDI and PFL after first claiming they were unmodeled. The correct mandatory state contributions total $443.11: $411.91 for PFL and $31.20 for Disability Benefits.
+us,scenario_082,payroll_tax,deepseek-v4-pro,llm_error,state_local_rule,False,The model included the $31.20 New York Disability Benefits contribution but entirely omitted the $411.91 Paid Family Leave contribution.
+us,scenario_082,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model understated New York Paid Family Leave at approximately $333 rather than $411.91. With the $31.20 Disability Benefits contribution, the state total is $443.11."
+us,scenario_082,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model stopped at the 7.65% federal FICA calculation and omitted both mandatory New York components. PFL and Disability Benefits add $411.91 and $31.20, respectively."
+us,scenario_082,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly subtracted $21,208 of employer-sponsored insurance premiums from FICA wages, reducing the payroll-tax base to $78,987. The trace applies Social Security, Medicare, PFL, and Disability Benefits to the full $100,195 of taxable wages, and the model also understated PFL."
+us,scenario_082,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly treated the listed employer-sponsored insurance premiums as a pretax reduction of FICA wages and taxed only $78,987. The correct Social Security and Medicare base is the full $100,195, with another $443.11 due for New York PFL and Disability Benefits."
+us,scenario_082,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $21,208 employer-sponsored insurance premiums from FICA-taxable wages. All four employee payroll-tax components use the full $100,195 wage base, including $411.91 of PFL and $31.20 of Disability Benefits."
+us,scenario_082,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model incorrectly used $78,987 after subtracting employer-sponsored insurance premiums from FICA wages. The correct base is $100,195, and the exact New York contributions are $411.91 for PFL and $31.20 for Disability Benefits."
+us,scenario_082,payroll_tax,glm-5.2,llm_error,state_local_rule,False,"The model used a 0.388% PFL rate and an incorrect $96,200 cap, yielding $373.26. The correct PFL contribution is $411.91; adding the correctly identified $31.20 Disability Benefits contribution produces $443.11 of New York payroll tax."
+us,scenario_082,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly asserted that New York has no mandatory employee payroll tax and also rounded federal FICA below its exact $7,664.92 amount. It omitted $411.91 of PFL and $31.20 of Disability Benefits."
+us,scenario_082,payroll_tax,gpt-5.4-nano,llm_error,state_local_rule,False,"The model approximated federal FICA instead of retaining the exact $7,664.92 and omitted all New York mandatory employee contributions. PFL and Disability Benefits add $443.11."
+us,scenario_082,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"The model correctly calculated federal FICA but estimated combined New York PFL and Disability Benefits at $385.73. The correct state total is $443.11, comprising $411.91 and $31.20."
+us,scenario_082,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,The model included the $31.20 New York Disability Benefits contribution but omitted the separate $411.91 Paid Family Leave contribution.
+us,scenario_082,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,The model calculated only federal Social Security and Medicare. It omitted mandatory New York PFL of $411.91 and Disability Benefits of $31.20.
+us,scenario_082,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model omitted both mandatory New York payroll taxes and also misstated the federal FICA total, which is $7,664.92 rather than $7,654.00. New York PFL and Disability Benefits add $443.11."
+us,scenario_082,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model used only the 7.65% federal FICA rate. It omitted $411.91 of New York Paid Family Leave and $31.20 of New York Disability Benefits contributions.
+us,scenario_082,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly subtracted $21,208 of employer-sponsored insurance premiums from FICA wages and used $78,987 instead of $100,195. It also omitted the $443.11 of mandatory New York PFL and Disability Benefits contributions."
+us,scenario_082,payroll_tax,grok-4.6,llm_error,state_local_rule,False,The model understated New York PFL at approximately $367 and incorrectly excluded the $31.20 Disability Benefits contribution. The correct New York total is $443.11.
+us,scenario_082,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model incorrectly deducted both employer-sponsored insurance premiums and the traditional 401(k) contribution from FICA wages, reducing the base to $78,806. The trace taxes the full $100,195 and also includes $411.91 of New York PFL and $31.20 of Disability Benefits."
+us,scenario_082,payroll_tax,inkling,llm_error,state_local_rule,False,"The model approximated New York PFL at $365 instead of $411.91. Combined with the exact $31.20 Disability Benefits contribution, New York payroll tax is $443.11, not approximately $396."
+us,scenario_082,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract."
+us,scenario_082,payroll_tax,minimax-m3,llm_error,state_local_rule,False,"The model stopped after federal Social Security and Medicare and rounded that subtotal to $7,665. It omitted $411.91 of New York PFL and $31.20 of Disability Benefits."
+us,scenario_082,payroll_tax,ox-alpha,llm_error,state_local_rule,False,The model calculated and rounded only federal Social Security and Medicare. It omitted the mandatory New York PFL and Disability Benefits contributions totaling $443.11.
+us,scenario_082,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model produced mutually inconsistent New York tax calculations and then inserted a negative $200 state adjustment despite payroll contributions being positive liabilities. The correct state calculation is $411.91 of PFL plus $31.20 of Disability Benefits, added to $7,664.92 of federal FICA."
+us,scenario_082,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model expressly excluded mandatory state payroll taxes, omitting $411.91 of New York PFL and $31.20 of Disability Benefits. Its submitted $7,752.42 also does not equal the $7,664.92 federal subtotal stated in its own reasoning."
 us,scenario_082,self_employment_tax,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_082,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model invented a roughly $396 nonrefundable child and dependent care credit reduction. This output retains the $5,442.17 main tax and adds $156.38 of supplemental tax; it does not subtract the refundable New York child and dependent care credit."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model equated New York taxable income with a miscomputed $62,149 federal taxable-income figure and used the federal standard deduction. New York instead starts from $117,585.15 of AGI and applies its own $11,200 standard deduction plus the $1,000 dependent exemption, yielding $105,385.15."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model treated the filer as single and used the $8,000 single standard deduction despite the dependent child establishing head-of-household filing. It also never performed the traced main-tax-plus-supplemental-tax calculation on $105,385.15 of taxable income."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used single filing status and an $8,000 standard deduction instead of head-of-household status, the $11,200 standard deduction, and the $1,000 dependent exemption. Its unexplained final adjustment also bypassed the $5,442.17 main tax plus $156.38 supplemental tax calculation."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an $8,000 deduction even while describing head-of-household filing and omitted the $1,000 dependent exemption. The applicable deductions reduce $117,585.15 to $105,385.15, after which the supplemental-tax step produces $5,598.55."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model calculated the return as single, itemized $9,917, and subtracted a $120 child-care credit. The return is head of household, uses the larger $11,200 standard deduction plus a $1,000 exemption, and adds $156.38 of supplemental tax without subtracting that refundable credit."
-us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"After using single-filer deductions and brackets, the model arbitrarily reduced its computed tax by about $1,393 for unspecified household or child credits. Those credits do not reduce this before-refundable-credits output, which adds the $156.38 supplemental tax to $5,442.17."
-us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model claimed $20,691 of itemized deductions, producing taxable income of $95,971. The allowable itemized amount is $11,169.24, so the larger $11,200 standard deduction applies; with the $1,000 exemption, taxable income is $105,385.15."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a $10,500 standard deduction and subtracted a $30 household credit. The applicable standard deduction is $11,200, taxable income is $105,385.15 after the $1,000 exemption, and the liability includes $156.38 of supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The wage-only estimate omitted taxable dividends, interest, gains and losses, and the other AGI components that produce New York AGI of $117,585.15. It therefore never reached $105,385.15 of taxable income or applied the supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The stated $84,178 taxable income is $21,207.15 too low. The trace yields $117,585.15 of AGI less $11,200 and $1,000, or $105,385.15, before applying the main and supplemental taxes."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated New York AGI at $96,454, excluding $21,131.15 of taxable income components. New York AGI is $117,585.15 and taxable income after the standard deduction and exemption is $105,385.15."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income far below the traced $105,385.15 and provides no component calculation supporting that reduction. Applying the $11,200 standard deduction and $1,000 exemption to $117,585.15 of AGI, then adding supplemental tax, yields $5,598.55."
-us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model's reference to allowable nonrefundable credits and its $3,591 result imply large unsupported credit reductions. The output requires $5,442.17 of main tax plus $156.38 of supplemental tax, with refundable child-related credits excluded from the subtraction."
-us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested variable, so the required numeric output was missing."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model gave a rounded estimate without applying the complete New York computation. On $105,385.15 of taxable income, the main tax is $5,442.17 and the required supplemental tax is $156.38."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $7,777 estimate is inconsistent with the applicable deduction choice and taxable-income base. The standard deduction of $11,200 exceeds $11,169.24 of itemized deductions, leaving $105,385.15 taxable and total tax of $5,598.55."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model reached taxable income within $8.85 of the traced amount but understated the resulting liability by $50.55. It failed to apply the exact New York main-tax calculation and $156.38 supplemental tax, which total $5,598.55."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The $2,954.80 estimate applies neither the full graduated tax on $105,385.15 nor the supplemental-tax calculation. The main tax alone is $5,442.17, before adding $156.38."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"Although the model used taxable income near the traced amount, it applied the New York rate schedule incorrectly. Taxable income of $105,385.15 produces $5,442.17 of main tax and $156.38 of supplemental tax, not $5,815."
-us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model overstated tax after identifying the applicable standard deduction. The exact schedule produces $5,442.17 of main tax and $156.38 of supplemental tax on $105,385.15, totaling $5,598.55."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The rounded $5,500 answer omitted the exact bracket and supplemental-tax computation. The two required components are $5,442.17 of main tax and $156.38 of supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated New York AGI by $21,131.15 and then subtracted a $120 child-care credit from this before-refundable-credits output. AGI is $117,585.15, taxable income is $105,385.15, and the refundable credit is not deducted."
-us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model allowed $2,000 of exemptions instead of the single $1,000 dependent exemption and subtracted a $120 New York child-care credit. This output excludes that refundable credit and instead adds the $156.38 supplemental tax to the $5,442.17 main tax."
-us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested variable, so the required numeric output was missing."
-us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model reached taxable income within $8.85 of the traced figure but used the wrong rate schedule and omitted the exact supplemental-tax treatment. The applicable computation is $5,442.17 of main tax plus $156.38 of supplemental tax."
-us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model offset ordinary income with the entire $33,000 long-term capital loss instead of applying the $3,000 annual net-capital-loss limit embedded in AGI. It also invoked child-related credits that do not erase this before-refundable-credits state liability."
-us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the full $30,711 net capital loss rather than limiting the deduction against ordinary income to $3,000, understating AGI by $27,711. It also used single status and produced an arithmetically unsupported increase after its claimed household credit."
-us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model overstated AGI at $139,586, used single-filer deductions, and then subtracted the refundable Empire State Child Credit and other unsupported credits. The required base is $105,385.15 of taxable income, with refundable credits excluded and $156.38 of supplemental tax added."
-us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model used an obsolete 33%-of-$1,000 Empire State Child Credit formula instead of starting with the 2026 $1,000 New York Child Tax Credit and applying the $709.50 AGI phaseout. It also understated the dependent-care credit, which is $3,000 × 0.6 × 0.2 = $360."
-us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that New York lacks refundable child and dependent-care credits and treated the income as disqualifying. The household receives a phaseout-adjusted $290.50 New York Child Tax Credit and a $360 refundable Child and Dependent Care Credit.
-us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly restricted the child credit for a child under age four to $110 and classified the New York dependent-care credit as nonrefundable at this AGI. The child qualifies for $290.50 after phaseout, and the refundable dependent-care credit is $360."
-us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated both New York credits as eliminated by income. At AGI $117,585.15, the child-credit phaseout leaves $290.50 and the applicable dependent-care percentage produces another $360."
-us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated the household's income as fully eliminating the New York child and dependent-care credits. The applicable phaseout and rate calculations leave $290.50 and $360, respectively."
-us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model used 33% of the federal $2,000 CTC with no New York phaseout, yielding $660 instead of the phaseout-adjusted $290.50. It also applied a 20% New York multiplier to the $600 federal CDCC instead of the applicable 60% multiplier, understating that credit as $120 rather than $360."
-us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,The model incorrectly concluded that income fully phases out the Empire State Child Credit and overlooked the refundable dependent-care credit. The trace's AGI calculation leaves $290.50 of child credit and generates a $360 dependent-care credit.
-us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a 20% New York multiplier to the $600 federal CDCC, but the applicable New York rate at AGI $117,585.15 is 60%, producing $360. It also omitted the $290.50 phaseout-adjusted New York Child Tax Credit."
-us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model used $330 instead of the phaseout-adjusted $290.50 child credit, understated the dependent-care credit at roughly $300 instead of $360, and added an inapplicable $63 NYC School Tax Credit. The stated household location is New York State, not New York City."
-us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned a flat $330 Empire State Child Credit rather than applying the 2026 $1,000 base and $709.50 phaseout, and it omitted the $360 dependent-care credit."
-us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model reduced the child credit to a $100 minimum instead of calculating the $290.50 amount remaining after phaseout. It also used a 20% state multiplier rather than the applicable 60% multiplier on the $600 federal CDCC, yielding $120 instead of $360."
-us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 110% dependent-care multiplier instead of the applicable 60%, producing $660 rather than $360. It also used a flat $330 child credit rather than the $290.50 amount after the AGI phaseout."
-us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model used a flat $330 Empire State Child Credit instead of the $1,000 base reduced by the $709.50 phaseout to $290.50. It omitted the separate $360 refundable dependent-care credit."
-us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,The model treated $330 as the complete refundable New York credit. It failed to apply the child-credit phaseout yielding $290.50 and omitted the $360 Child and Dependent Care Credit.
+us,scenario_082,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model subtracted an estimated $396 child and dependent care credit from this output. The trace applies no such nonrefundable reduction here and instead adds the $156.38 supplemental tax to $5,442.17 of regular tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model equated New York taxable income with a $62,149 federal-taxable-income estimate and used a federal deduction framework. New York starts from $117,585.15 of AGI and applies its own $11,200 standard deduction and $1,000 exemption, yielding $105,385.15."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model filed the parent as single and used the $8,000 single standard deduction despite the parent qualifying as head of household with a dependent child. The calculation requires the head-of-household schedule, an $11,200 standard deduction, and the $156.38 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model used single filing status and the $8,000 single standard deduction. The correct head-of-household calculation uses the $11,200 standard deduction, a $1,000 dependent exemption, and then adds $156.38 of supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model used an approximately $8,000 deduction while calling the filer head of household. The applicable New York head-of-household standard deduction is $11,200, and the final liability also includes $156.38 of supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model explicitly calculated the return as single, itemized $9,917, and subtracted a $120 childcare credit. The required head-of-household computation instead uses the larger $11,200 standard deduction, subtracts the $1,000 dependent exemption, and adds the $156.38 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used single-filer deductions and brackets, then introduced unspecified credits to reduce its computed tax to $4,800. The return uses head-of-household status, the $11,200 standard deduction, the $1,000 exemption, and the supplemental-tax calculation."
+us,scenario_082,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $20,691 of itemized expenses, reducing taxable income to $95,971. New York uses the larger applicable deduction of $11,200 plus the $1,000 dependent exemption, leaving $105,385.15, not $95,971."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $10,500 deduction rather than the $11,200 head-of-household standard deduction and subtracted a $30 household credit. It also failed to apply the $156.38 supplemental tax that raises regular tax of $5,442.17 to $5,598.55."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's wage-based estimate omits the traced New York tax base, which includes dividends and the federally limited capital loss in $117,585.15 of AGI. Applying the head-of-household deductions and both regular and supplemental tax produces $5,598.55, not $4,380."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used New York taxable income of $84,178. The traced tax base is $105,385.15 after subtracting only the $11,200 standard deduction and $1,000 exemption from $117,585.15 of AGI."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated New York AGI as $96,454 and taxable income as $79,254. The capital loss is limited in AGI rather than fully offsetting income, producing $117,585.15 of AGI and $105,385.15 of taxable income."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $4,220 estimate does not apply the traced calculation to $105,385.15 of taxable income. Regular tax is $5,442.17 and the required supplemental tax adds $156.38."
+us,scenario_082,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model describes a liability after unspecified nonrefundable credits, producing an excessive reduction to $3,591. This output retains $5,442.17 of regular tax and adds $156.38 of supplemental tax, with no traced credit reduction."
+us,scenario_082,state_income_tax_before_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The rounded estimate failed to complete the New York rate calculation. On taxable income of $105,385.15, regular tax is $5,442.17 and supplemental tax is $156.38, totaling $5,598.55."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $7,777 estimate is inconsistent with the applicable deduction and tax schedule. New York taxable income is $105,385.15 after the $11,200 standard deduction and $1,000 exemption, on which regular and supplemental tax total $5,598.55."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model reached nearly the correct taxable income but stopped at an incorrect bracket estimate and stated that no further component applied. It omitted New York's $156.38 supplemental tax; the traced regular tax is $5,442.17 before that addition."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The $2,954.80 estimate substantially underapplies New York's head-of-household tax schedule. Taxable income of $105,385.15 produces $5,442.17 of regular tax plus $156.38 of supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used roughly the correct taxable-income range but misapplied the New York tax schedule. The required components are $5,442.17 of regular tax and $156.38 of supplemental tax, not $5,815."
+us,scenario_082,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model applied excessive tax to income after the $11,200 deduction and did not identify the separate supplemental-tax computation. The traced amounts are $5,442.17 of regular tax and $156.38 of supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The rounded $5,500 estimate omitted the exact bracket and supplemental-tax steps. New York regular tax is $5,442.17 and supplemental tax is $156.38, yielding $5,598.55."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $96,454 by failing to enforce the $3,000 limit on the net capital-loss deduction. It therefore taxed only $83,354 and also subtracted a $120 childcare credit instead of using the traced $105,385.15 tax base and supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model closely matched the taxable-income derivation but used an incomplete simplified bracket formula. The trace calculates $5,442.17 of regular tax and then adds the omitted $156.38 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model deducted $2,000 of exemptions instead of the traced $1,000 dependent exemption and then subtracted a $120 childcare credit. The correct taxable income is $105,385.15, followed by $5,442.17 of regular tax and $156.38 of supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_082,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model reached nearly the traced taxable income but applied the wrong head-of-household schedule through a stated 5.75% bracket. The correct regular tax is $5,442.17, followed by a $156.38 supplemental-tax addition."
+us,scenario_082,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated the $33,000 long-term capital loss as eliminating much of the tax base and invoked child credits to reduce liability to zero. The net capital-loss deduction is limited to $3,000 in AGI, leaving $117,585.15 of AGI and $105,385.15 of taxable income."
+us,scenario_082,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model used an $8,000 deduction while labeling the calculation head of household, overstating taxable income, and then subtracted a $120 childcare credit. The applicable standard deduction is $11,200, followed by a $1,000 exemption and the separate $156.38 supplemental tax."
+us,scenario_082,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the full $30,711 net capital loss instead of limiting the loss entering federal AGI to $3,000. It also used single filing status and described a household credit as increasing tax, producing an internally inconsistent result."
+us,scenario_082,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model overstated AGI at $139,586, used the $8,000 single deduction, and subtracted the Empire State Child Credit and other child credits from this pre-refundable-credit output. The traced calculation instead uses $117,585.15 of AGI, the $11,200 head-of-household deduction, a $1,000 exemption, and the supplemental tax."
+us,scenario_082,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model used an obsolete $330 federal-CTC-percentage shortcut instead of starting with New York's $1,000 child amount and subtracting the $709.50 AGI phase-out, which yields $290.50. It also estimated only $150 of dependent-care credit instead of applying $3,000 × 0.6 × 0.2 = $360."
+us,scenario_082,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model incorrectly asserted that New York lacks refundable child and child-care credits. This household receives the refundable $290.50 New York child credit and $360 New York child and dependent care credit despite being ineligible for the EITC.
+us,scenario_082,state_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,The model wrongly excluded the child and dependent care credit at this AGI and substituted a $110 under-four child amount. The applicable calculations are a $290.50 phased child credit and a refundable $360 dependent-care credit.
+us,scenario_082,state_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated both New York credits as eliminated at this income. The child credit is only partially phased out to $290.50, and the refundable dependent-care formula still yields $360."
+us,scenario_082,state_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly treated income as eliminating the New York child and dependent-care credits. At AGI $117,585.15, the respective refundable amounts are $290.50 and $360."
+us,scenario_082,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly netted the full $33,000 long-term capital loss in estimating AGI and then calculated the child credit as 33% of a $2,000 federal CTC. PolicyEngine uses AGI $117,585.15, producing a $290.50 child credit after phase-out, while the dependent-care formula is $3,000 × 0.6 × 0.2 = $360 rather than $120."
+us,scenario_082,state_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model treated the Empire State child credit as fully phased out and omitted the dependent-care credit. The AGI-based child-credit reduction leaves $290.50, and the refundable dependent-care calculation adds $360."
+us,scenario_082,state_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model reduced the New York dependent-care credit to 20% of the $600 federal CDCC, yielding $120, instead of applying $3,000 × 0.6 × 0.2 = $360. It also omitted the $290.50 child credit remaining after phase-out."
+us,scenario_082,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,The model invented a $63 NYC school tax credit without facts establishing NYC residence and used incorrect amounts for both actual components. The correct state components are $290.50 of child credit and $360 of dependent-care credit.
+us,scenario_082,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model used a flat $330 Empire State child credit rather than the $1,000 base less the $709.50 phase-out, and it omitted the $360 dependent-care credit. Those components total $650.50."
+us,scenario_082,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model replaced the phased child credit with a $100 minimum and computed only $120 of dependent-care credit. The correct amounts are $290.50 and $360 because the latter uses $3,000 × 0.6 × 0.2."
+us,scenario_082,state_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 110% New York dependent-care rate and a flat $330 child credit. At AGI $117,585.15, the applicable dependent-care rate is 0.6 in the traced formula, yielding $360, and the phased child credit is $290.50."
+us,scenario_082,state_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The model used a flat $330 child-credit amount and omitted the refundable dependent-care credit. The child-credit phase-out yields $290.50, and dependent-care expenses generate another $360."
+us,scenario_082,state_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model reduced the entire result to a $330 Empire State child credit. It missed the $709.50 phase-out from the $1,000 child amount and the separate $360 dependent-care credit."
 us,scenario_082,state_refundable_credits,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, so the required output could not be parsed."
-us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked both refundable credits triggered by the qualifying one-year-old child and listed childcare expenses. Those facts produce a $290.50 New York Child Tax Credit and a $360 Child and Dependent Care Credit.
-us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model wrongly required explicit credit-specific eligibility inputs rather than deriving eligibility from the qualifying child, AGI, and childcare expenses. Those inputs generate refundable credits of $290.50 and $360."
-us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a 20% New York multiplier on the $600 federal CDCC instead of the applicable 60%, understating the dependent-care credit by $240. It also incorrectly treated the New York Child Tax Credit as fully eliminated instead of retaining $290.50 after phaseout."
-us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model awarded the full $1,000 child-credit base without applying the $709.50 AGI phaseout. It also omitted the separate $360 refundable dependent-care credit."
-us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly obtained the $290.50 child credit but applied a 20% state multiplier to the $600 federal CDCC, producing $120. The applicable New York multiplier is 60%, so the dependent-care credit is $360."
-us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly excluded the one-year-old child from the New York child-credit age range and treated income as eliminating all refundable credits. The child qualifies for $290.50 after phaseout, and the childcare expenses produce a $360 refundable dependent-care credit."
-us,scenario_082,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model failed to recognize that the qualifying child and $9,600 of childcare expenses trigger two refundable New York credits. Their calculated amounts are $290.50 and $360."
-us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model incorrectly declared the federal CTC fully phased out and then used that conclusion to eliminate the New York child credit. At the trace AGI, the New York phaseout leaves $290.50, and the model also omitted the independent $360 dependent-care credit."
-us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated the absence of federal EITC as eliminating all state refundable credits. It omitted the $290.50 New York Child Tax Credit and the $360 refundable Child and Dependent Care Credit, neither of which depends on receiving EITC."
-us,scenario_082,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model incorrectly treated the New York dependent-care credit as merely absorbed by tax liability even though it is refundable in this calculation. It also omitted the phaseout-adjusted $290.50 child credit; the dependent-care amount is $360.
+us,scenario_082,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model incorrectly concluded that the facts trigger no refundable New York credits. The qualifying child and childcare expenses produce a $290.50 phased child credit and a $360 dependent-care credit.
+us,scenario_082,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated eligibility as requiring explicit credit-specific inputs beyond the supplied child, income, and childcare facts. Those facts generate the $290.50 New York child credit and $360 dependent-care credit."
+us,scenario_082,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model computed the dependent-care component as 20% of the $600 federal credit rather than $3,000 × 0.6 × 0.2, understating it from $360 to $120. It also incorrectly eliminated the child credit, which remains $290.50 after phase-out."
+us,scenario_082,state_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model took the $1,000 child-credit base as the final refundable total and failed to subtract the $709.50 AGI phase-out. It also omitted the separate $360 dependent-care credit."
+us,scenario_082,state_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model correctly derived the $290.50 child credit but understated the dependent-care credit as $120. Applying the traced 0.6 New York rate and 0.2 federal rate to $3,000 yields $360."
+us,scenario_082,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model wrongly excluded the one-year-old child from the New York child credit and treated income as eliminating all refundable credits. The child generates $290.50 after phase-out, and the childcare expenses generate $360."
+us,scenario_082,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted both qualifying New York pathways. The child credit is $290.50 after phase-out, and the child and dependent care credit is $360."
+us,scenario_082,state_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated the federal CTC and associated New York child credit as fully phased out at this income and omitted the dependent-care pathway. The New York child calculation leaves $290.50, and childcare expenses produce $360."
+us,scenario_082,state_refundable_credits,grok-4.6,llm_error,credit_phaseout,False,"The model substituted a $100 minimum for the child credit instead of applying the $709.50 phase-out to the $1,000 base, which leaves $290.50. It also used $120 for dependent care instead of $3,000 × 0.6 × 0.2 = $360."
+us,scenario_082,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model considered only the EITC pathway and overlooked New York's refundable child and dependent-care credits. Those credits equal $290.50 and $360 respectively.
+us,scenario_082,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model incorrectly treated the New York child-care credit as absorbed by tax liability rather than refundable and omitted the child credit. The refundable amounts are $360 for dependent care and $290.50 for the phased child credit.
 us,scenario_082,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for state_refundable_credits, so the required output could not be parsed."
-us,scenario_082,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model substituted a $100 minimum for the child credit instead of applying the $709.50 phaseout to the $1,000 base, which leaves $290.50. It also used a 20% New York multiplier instead of the applicable 60% multiplier on the $600 federal CDCC, yielding $120 rather than $360."
-us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly used income and pre-credit tax liability to eliminate refundable credits. Refundability makes tax liability irrelevant to the allowed amounts, and the applicable income calculations leave $290.50 of child credit plus $360 of dependent-care credit."
-us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model considered only the New York earned-income-credit limit and incorrectly concluded that no other refundable credits apply. The qualifying child and childcare expenses produce a $290.50 child credit and a $360 dependent-care credit.
-us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,other,False,The model conflated the Empire State Child Credit and New York Child Tax Credit as separate partially refundable items and produced an unsupported combined estimate. The trace contains one child-credit component of $290.50 plus a distinct $360 Child and Dependent Care Credit.
+us,scenario_082,state_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model incorrectly reduced the child credit to a $100 minimum instead of the $290.50 left after the traced AGI phase-out. It also understated dependent care as $120 rather than $360 by failing to apply the 0.6 New York rate to the $3,000 expense base before the 0.2 federal rate."
+us,scenario_082,state_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,The model incorrectly linked refundable-credit availability to having little or no pre-credit state tax liability and treated income as eliminating the child credit. Refundability permits the $290.50 phased child credit and $360 dependent-care credit independently of remaining tax liability.
+us,scenario_082,state_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model applied a 5% reduction to all AGI above $75,000 and therefore erased the child credit. The traced phase-out is $709.50 from a $1,000 base, leaving $290.50, and the model also omitted the $360 dependent-care credit."
+us,scenario_082,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model stopped after ruling out the New York EITC and ignored the refundable child and dependent-care pathways. They yield $290.50 and $360 respectively.
+us,scenario_082,state_refundable_credits,qwen3.8-max,llm_error,other,False,The model conflated the Empire State child credit with a separate New York child tax credit and produced an unsupported combined estimate. The actual components are one $290.50 child credit after phase-out and a distinct $360 child and dependent care credit.
 us,scenario_083,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly assumed the individual provisions of the Tax Cuts and Jobs Act expired for 2026 and used an $8,328 single-filer standard deduction. The applicable 2026 standard deduction exceeds $11,010, so taxable income and the resulting 10% bracket tax are both zero."
 us,scenario_083,federal_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly determined that the age-20 childless filer fails the EITC age requirement and that no other refundable credit applies, but then submitted $540 instead of carrying the resulting $0 into the output."
 us,scenario_083,federal_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly concluded that the filer is too young for the childless EITC and has no qualifying child for refundable CTC, but its $538 output contradicts its own all-zero component calculation."
@@ -5088,69 +5343,75 @@ us,scenario_083,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's subm
 us,scenario_083,payroll_tax,minimax-m3,llm_error,other,False,The model correctly derived $539.33 and then improperly changed it to $538 under the label of rounding. Ordinary currency rounding preserves $539.33; it does not reduce the result by $1.33.
 us,scenario_083,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The submitted $649.62 is inconsistent with applying employee Social Security and Medicare taxes to the stated $7,050 wages. The correct 7.65% combined tax on that wage base is $539.33, so the answer used an inflated base or rate unsupported by the household facts."
 us,scenario_083,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly classified the $3,960 of rental income as self-employment income and applied the 92.35% adjustment and 15.3% tax rate to it. Ordinary rental income is excluded from net earnings from self-employment, leaving no self-employment earnings subject to the $400 threshold and yielding $0 of self-employment tax."
-us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model never applied Texas's $5,000 SNAP countable-resource limit to the listed $6,620 bank balance. That excess makes the household ineligible, so its subsequent earned-income deduction, standard deduction, and allotment calculation cannot produce a benefit."
-us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model explicitly treated Texas broad-based categorical eligibility as waiving the asset test. Texas instead imposes a $5,000 resource limit under that pathway, and the $6,620 bank balance disqualifies the household before benefit computation."
-us,scenario_083,snap,claude-opus-5,llm_error,asset_resource,False,"The model calculated an allotment solely from gross and net income and omitted the resource-eligibility step. The household's $6,620 bank assets exceed Texas's $5,000 SNAP resource limit, yielding no eligibility and no benefit."
-us,scenario_083,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model skipped Texas's $5,000 resource test and proceeded to estimate a positive income-based allotment. The listed $6,620 bank balance disqualifies the household, and the mortgage balance supplies neither an asset-test exception nor a reported shelter payment deduction."
-us,scenario_083,snap,gemini-3-flash-preview,llm_error,asset_resource,False,"The model applied the earned-income and standard deductions directly to the allotment formula without testing countable resources. The $6,620 bank balance exceeds Texas's $5,000 SNAP resource limit, so the correct computation stops at ineligibility."
-us,scenario_083,snap,glm-5.2,llm_error,asset_resource,False,"The model subtracted income deductions and 30% of net income from a maximum allotment but omitted resource eligibility. Texas's $5,000 limit is exceeded by the household's $6,620 bank assets, forcing the SNAP amount to zero."
-us,scenario_083,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model treated countable monthly income as the only binding eligibility input and calculated a positive allotment. It failed to apply Texas's $5,000 resource limit to the $6,620 bank balance, which makes the household ineligible."
-us,scenario_083,snap,grok-4.5,llm_error,asset_resource,False,"The model explicitly asserted that Texas broad-based categorical eligibility waives assets. Texas retains a $5,000 countable-resource limit under that pathway, so the household's $6,620 bank balance eliminates SNAP eligibility before the net-income formula."
-us,scenario_083,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model moved from income deductions to an annual allotment without applying the resource test. Because $6,620 in bank assets exceeds Texas's $5,000 SNAP resource limit, the household is ineligible regardless of its income-based allotment calculation."
-us,scenario_084,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model correctly identified that income and assets exceed the SSI-related limits, then overrode those limits by treating disability, blindness, SSDI receipt, and surviving-spouse status as automatic categorical eligibility. None of those facts creates a qualifying North Carolina Medicaid pathway without satisfying the applicable financial requirements."
-us,scenario_084,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model’s own reasoning found that the head exceeds the aged/blind/disabled income limit and has no MAGI parent or child category, but it submitted Yes anyway. Those findings yield no qualifying Medicaid category and therefore an eligibility value of 0."
-us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented medically needy spend-down eligibility by subtracting the listed annual medical expenses from income. The trace assigns no Medicaid category, and those expenses do not establish a qualifying North Carolina spend-down pathway for this head under PolicyEngine’s rules."
-us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated being disabled and blind as sufficient for a disability-related Medicaid pathway while leaving the required income and category tests unresolved. The head qualifies through no Medicaid category, so those statuses alone yield no eligibility."
-us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security disability income from Medicaid MAGI merely because it is nontaxable for ordinary income-tax purposes. Medicaid MAGI includes the applicable nontaxable Social Security amount, producing 1.56 times FPL rather than the model’s $8,405 estimate and placing the head above the expansion threshold."
-us,scenario_084,head_medicare_eligible,claude-fable-5,llm_error,health_coverage,False,"The model converted full-year SSDI receipt into completion of Medicare's 24-month SSDI entitlement period. Constancy during 2026 establishes only that year's status, not the required prior duration, so the under-65 disability pathway was not established."
-us,scenario_084,head_medicare_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model treated disability plus current SSDI receipt as sufficient for Medicare and never established 24 months of SSDI entitlement. At age 62, neither those facts nor the full-year constancy instruction establishes an under-65 Medicare pathway."
-us,scenario_084,head_medicare_eligible,claude-opus-4.7,llm_error,health_coverage,False,The model inferred that the 24-month SSDI waiting period had elapsed from disability status being constant throughout the tax year. A single year of constant status does not establish the required benefit-entitlement history.
-us,scenario_084,head_medicare_eligible,claude-opus-4.8,llm_error,health_coverage,False,The model expressly assumed established disability-benefit receipt satisfied the 24-month SSDI rule. The prompt supplies current annual SSDI income but no entitlement start date or completed waiting period.
-us,scenario_084,head_medicare_eligible,claude-opus-5,llm_error,health_coverage,False,"The model treated receipt of Social Security disability benefits as immediate qualification through Medicare's disability pathway. Under age 65, current SSDI receipt alone does not establish completion of the 24-month entitlement period."
-us,scenario_084,head_medicare_eligible,claude-sonnet-5,llm_error,health_coverage,False,The model correctly named the 24-month rule but treated being disabled and receiving SSDI during 2026 as proof that the period had elapsed. No duration before or during the year establishes completion of that waiting period.
-us,scenario_084,head_medicare_eligible,deepseek-v4-pro,llm_error,health_coverage,False,"The model invented at least 24 months of SSDI receipt from the person's age and current benefit receipt. Age 62 implies no such benefit history, so the disability-based Medicare pathway was not established."
-us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,llm_error,health_coverage,False,The model treated disability and current Social Security disability income as sufficient for under-65 Medicare eligibility. It omitted the required SSDI entitlement duration and identified no ALS or end-stage renal disease pathway.
-us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,health_coverage,False,The model equated receipt of Social Security disability income with Medicare eligibility. Current SSDI income does not by itself establish the 24-month entitlement period required before age 65.
-us,scenario_084,head_medicare_eligible,gemini-3.5-flash,llm_error,health_coverage,False,The model treated disabled status and SSDI receipt as automatically qualifying. It failed to apply the 24-month SSDI entitlement requirement for an individual under age 65.
-us,scenario_084,head_medicare_eligible,gemini-3.6-flash,llm_error,health_coverage,False,"The model made disability plus SSDI receipt an immediate Medicare pathway. At age 62, the missing fact is completion of the 24-month SSDI entitlement period, which cannot be inferred from annual income."
-us,scenario_084,head_medicare_eligible,gemini-3.7-flash,llm_error,health_coverage,False,The model assumed that being disabled and receiving SSDI itself confers Medicare eligibility. It did not establish the required 24 months of SSDI entitlement or any separate qualifying medical pathway.
-us,scenario_084,head_medicare_eligible,glm-5.2,llm_error,health_coverage,False,The model misstated PolicyEngine's under-65 rule by treating disabled status and SSDI income as sufficient. The supplied facts do not establish completion of the Medicare waiting period or another qualifying medical condition.
-us,scenario_084,head_medicare_eligible,gpt-5.4-mini,llm_error,health_coverage,False,"The model inferred Medicare eligibility from disability and surviving-spouse status. Surviving-spouse status is not an under-65 Medicare qualification, and disability alone does not establish the required SSDI entitlement duration."
-us,scenario_084,head_medicare_eligible,gpt-5.4-nano,llm_error,health_coverage,False,The model treated disabled status and SSDI income as the additional facts needed for PolicyEngine to mark eligibility. Those facts do not state that the 24-month SSDI entitlement period was completed.
-us,scenario_084,head_medicare_eligible,gpt-5.5,llm_error,health_coverage,False,The model used disability and current SSDI income to bypass the age-65 criterion. It omitted the required duration of SSDI entitlement for the under-65 disability pathway.
-us,scenario_084,head_medicare_eligible,gpt-5.6-luna,llm_error,health_coverage,False,The model treated Social Security disability income as automatically creating Medicare eligibility before age 65. Current receipt does not establish completion of the 24-month SSDI entitlement period.
-us,scenario_084,head_medicare_eligible,gpt-5.6-sol,llm_error,health_coverage,False,The model applied the disability pathway solely from disabled status and SSDI receipt. It failed to require evidence that the 24-month SSDI entitlement period had elapsed.
-us,scenario_084,head_medicare_eligible,grok-4.3,llm_error,health_coverage,False,The model asserted sufficient SSDI duration even though the prompt provides no benefit start date or prior entitlement history. The required 24-month period cannot be inferred from the annual payment amount.
-us,scenario_084,head_medicare_eligible,grok-4.5,llm_error,health_coverage,False,The model declared the 24-month SSDI waiting period satisfied from current SSDI income and disability. Neither fact establishes how long the person has been entitled to SSDI.
-us,scenario_084,head_medicare_eligible,kimi-k3,llm_error,health_coverage,False,The model treated disability and SSDI receipt as sufficient Medicare qualification under age 65. It omitted the 24-month SSDI entitlement condition and identified no alternative under-65 pathway.
-us,scenario_084,head_medicare_eligible,minimax-m3,llm_error,health_coverage,False,"The model explicitly assumed the 24-month waiting period was satisfied from the two annual disability-income amounts. Payment amounts do not establish entitlement duration, and the prompt supplies no start date."
-us,scenario_084,head_medicare_eligible,qwen-3.7-max,llm_error,health_coverage,False,The model converted constant disability and SSDI receipt throughout 2026 into completion of a 24-month waiting period. The constancy instruction covers only the tax year and does not add the necessary prior-year entitlement history.
+us,scenario_083,snap,claude-opus-4.7,llm_error,asset_resource,False,"The model applied the gross- and net-income tests but omitted Texas’s SNAP resource test; the $6,620 bank balance disqualifies the household. Its submitted $2,718 also contradicts every benefit calculation in its reasoning, but any positive allotment is precluded by the asset limit."
+us,scenario_083,snap,claude-opus-4.8,llm_error,asset_resource,False,"The model explicitly treated Texas broad-based categorical eligibility as waiving the asset test. Texas retains a SNAP resource limit, and the household’s $6,620 bank balance exceeds it, producing no benefit."
+us,scenario_083,snap,claude-opus-5,llm_error,asset_resource,False,"The model went directly from low monthly income to the allotment formula and omitted the resource-eligibility screen. The $6,620 bank balance exceeds Texas’s SNAP resource limit, so the household is ineligible regardless of its calculated net income."
+us,scenario_083,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model omitted Texas’s SNAP resource test, under which the $6,620 bank balance makes the household ineligible. It also invented a shelter deduction from the mortgage balance even though no mortgage payment, property tax, insurance, utility, or other shelter expense was listed."
+us,scenario_083,snap,gemini-3-flash-preview,llm_error,asset_resource,False,"The model calculated an allotment solely from income deductions and never applied Texas’s SNAP resource limit. The $6,620 bank balance fails that eligibility screen, so the maximum-allotment-minus-contribution calculation never produces a payable benefit."
+us,scenario_083,snap,glm-5.2,llm_error,asset_resource,False,"The model performed the net-income allotment calculation without checking countable resources. The household’s $6,620 bank balance exceeds Texas’s SNAP resource limit and makes the benefit zero before the income contribution is calculated."
+us,scenario_083,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model applied the earned-income and standard deductions and then computed an allotment, omitting the resource-eligibility test. Texas’s SNAP resource limit is exceeded by the $6,620 bank balance, so no allotment is payable."
+us,scenario_083,snap,grok-4.5,llm_error,asset_resource,False,"The model explicitly asserted that Texas BBCE waives assets. Texas retains a SNAP resource limit, and the household’s $6,620 bank balance exceeds it, making the household ineligible despite passing the income test."
+us,scenario_083,snap,grok-4.6,llm_error,asset_resource,False,"The model checked BBCE income and the ABAWD work rule but omitted Texas’s resource limit. The $6,620 bank balance fails the resource test, so passing the work and income screens does not generate a benefit."
+us,scenario_083,snap,ox-alpha,llm_error,asset_resource,False,"The model treated passage of the gross-income test as sufficient to proceed to the net-income allotment formula and ignored countable assets. The $6,620 bank balance exceeds Texas’s SNAP resource limit, making the household ineligible."
+us,scenario_083,snap,qwen-3.7-max,llm_error,asset_resource,False,"The model calculated a positive annual allotment without applying Texas’s SNAP resource test. The household’s $6,620 bank balance exceeds the applicable resource limit, so the income-based benefit calculation is moot."
+us,scenario_084,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness, disability, and SSDI receipt as categorical Medicaid eligibility despite acknowledging that the head exceeds the SSI-related income and resource limits. None of those facts creates unconditional North Carolina Medicaid eligibility, and the head satisfies no qualifying pathway."
+us,scenario_084,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly reasoned that income exceeds the North Carolina aged/blind/disabled limit and that no MAGI parent or child category applies, but then submitted eligible. Its numeric answer directly contradicts its own determination that the head satisfies no disability or MAGI pathway."
+us,scenario_084,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model invented medically needy spend-down eligibility without applying North Carolina's full financial test. Even subtracting all $6,754 of listed premiums and medical expenses from $33,233 leaves $26,479, and the model also ignored the $56,500 in assets; the head therefore does not enter a medically needy pathway."
+us,scenario_084,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated disability and blindness as sufficient for Medicaid while leaving the required income, resource, and coverage-group tests unapplied. PolicyEngine assigns no Medicaid category because those statuses alone do not establish eligibility."
+us,scenario_084,head_medicaid_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model excluded SSDI from Medicaid MAGI merely because it considered the benefit nontaxable. Medicaid MAGI includes Social Security benefits through its program-specific adjustment, producing 1.56 times FPL rather than approximately $8,405 and placing the head above the expansion limit."
+us,scenario_084,head_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model calculated expansion MAGI as approximately $8,405 by omitting the head's Social Security disability income. The Medicaid MAGI computation yields 1.56 times FPL, so the head exceeds North Carolina's expansion threshold and has no other qualifying category."
+us,scenario_084,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted constant disability and SSDI receipt during 2026 into satisfaction of the 24-month Medicare waiting period. A full tax year establishes only 12 months of constant status, so the unstated prior entitlement duration remains zero and the age-62 head is not eligible."
+us,scenario_084,head_medicare_eligible,claude-haiku-4.5,llm_error,age_disability,False,"The model treated disability plus current SSDI receipt as enough to establish Medicare eligibility while citing the separate 24-month requirement. No 24-month entitlement history is listed, so the under-65 disability pathway is not established."
+us,scenario_084,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model incorrectly inferred that year-long constant disability status satisfies the 24-month SSDI entitlement period. The facts provide no prior SSDI duration, leaving the 62-year-old below the age-based threshold without a proven disability pathway."
+us,scenario_084,head_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model expressly assumed established disability-benefit receipt and thereby supplied the missing 24-month SSDI history. Unlisted duration must be treated as zero, so neither age 65 nor the disability waiting-period condition is met."
+us,scenario_084,head_medicare_eligible,claude-opus-5,llm_error,age_disability,False,"The model treated disabled status and Social Security disability benefits as immediately conferring Medicare eligibility. For an individual under 65, those facts do not establish completion of the required disability-entitlement waiting period."
+us,scenario_084,head_medicare_eligible,claude-sonnet-5,llm_error,age_disability,False,"The model cited the 24-month waiting period but concluded it was met from current SSDI receipt alone. The prompt gives no 24-month entitlement history, and constant facts for 2026 do not supply it."
+us,scenario_084,head_medicare_eligible,deepseek-v4-pro,llm_error,age_disability,False,"The model invented at least 24 months of SSDI receipt from the person's age and current benefit. SSDI receipt at age 62 does not encode its start date, so the disability-based Medicare waiting period is not satisfied by the stated facts."
+us,scenario_084,head_medicare_eligible,gemini-3-flash-preview,llm_error,age_disability,False,"The model equated disability and receipt of Social Security disability income with immediate Medicare eligibility. It omitted the required under-65 disability entitlement history, while the head is only 62."
+us,scenario_084,head_medicare_eligible,gemini-3.1-pro-preview,llm_error,age_disability,False,The model treated receipt of Social Security disability income as sufficient by itself. Current SSDI income does not establish completion of the 24-month entitlement waiting period required for Medicare before age 65.
+us,scenario_084,head_medicare_eligible,gemini-3.5-flash,llm_error,age_disability,False,"The model collapsed SSDI benefit eligibility and Medicare eligibility into the same condition. The latter requires the specified entitlement duration for this under-65 disability pathway, which is absent from the household facts."
+us,scenario_084,head_medicare_eligible,gemini-3.6-flash,llm_error,age_disability,False,"The model assumed that disabled status plus SSDI receipt automatically makes an under-65 person Medicare eligible. It failed to apply the disability-entitlement waiting-period condition, for which no qualifying history is listed."
+us,scenario_084,head_medicare_eligible,gemini-3.7-flash,llm_error,age_disability,False,"The model treated disability and SSDI receipt as directly qualifying for Medicare. Because the head is 62, eligibility requires a completed under-65 disability pathway, and the necessary 24-month entitlement history is not stated."
+us,scenario_084,head_medicare_eligible,glm-5.2,llm_error,age_disability,False,The model incorrectly asserted that PolicyEngine makes current disability and SSDI receipt sufficient for an under-65 person. It omitted the qualifying entitlement-duration condition and therefore returned eligibility at age 62.
+us,scenario_084,head_medicare_eligible,gpt-5.4-mini,llm_error,age_disability,False,"The model treated disability and surviving-spouse status as proof of Medicare eligibility before age 65. Surviving-spouse status does not replace the missing disability-entitlement waiting period, and no other qualifying Medicare pathway is listed."
+us,scenario_084,head_medicare_eligible,gpt-5.4-nano,llm_error,age_disability,False,"The model inferred SSDI-based Medicare entitlement merely from disabled status and disability income. Those inputs do not state that the required 24 months of entitlement have elapsed, so PolicyEngine does not mark this 62-year-old eligible."
+us,scenario_084,head_medicare_eligible,gpt-5.5,llm_error,age_disability,False,"The model treated disability and SSDI income as sufficient support for the under-65 pathway. It skipped the required entitlement-duration test, which the supplied facts do not satisfy."
+us,scenario_084,head_medicare_eligible,gpt-5.6-luna,llm_error,age_disability,False,"The model made SSDI receipt an automatic trigger for Medicare eligibility. For a 62-year-old, the missing 24-month disability-entitlement history prevents that conclusion."
+us,scenario_084,head_medicare_eligible,gpt-5.6-sol,llm_error,age_disability,False,The model applied the disability pathway without testing its waiting-period requirement. Current disability and SSDI receipt do not establish the required prior entitlement duration for someone under 65.
+us,scenario_084,head_medicare_eligible,grok-4.3,llm_error,age_disability,False,"The model asserted a sufficient SSDI duration even though none is provided. Under the instruction that unlisted facts are false or zero, the 24-month waiting-period condition is not met."
+us,scenario_084,head_medicare_eligible,grok-4.5,llm_error,age_disability,False,"The model declared the 24-month SSDI waiting period satisfied solely from current disability and SSDI income. The prompt contains no such duration, so the under-65 Medicare pathway fails."
+us,scenario_084,head_medicare_eligible,grok-4.6,llm_error,age_disability,False,The model invoked eligibility after the SSDI waiting period without establishing that the period elapsed. Disability-benefit receipt during the stated year does not supply 24 months of prior entitlement.
+us,scenario_084,head_medicare_eligible,kimi-k3,llm_error,age_disability,False,"The model treated SSDI receipt as immediately qualifying an under-65 disabled person for Medicare. It omitted the 24-month entitlement condition, which is not established by the listed facts."
+us,scenario_084,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model explicitly assumed the 24-month waiting period was satisfied from the two annual disability-income amounts. Benefit amounts reveal no start date or entitlement duration, so that assumption improperly creates the missing eligibility fact."
+us,scenario_084,head_medicare_eligible,qwen-3.7-max,llm_error,age_disability,False,"The model converted constant SSDI receipt throughout the tax year into completion of a 24-month waiting period. Year-long constancy supplies only 12 months and does not establish earlier entitlement, leaving the age-62 head ineligible."
 us,scenario_084,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model calculated that 30% of monthly net income exceeded the $298 maximum allotment, correctly producing $0, then discarded that result and asserted an unsupported $183 monthly benefit. It misapplied the minimum-benefit rule, which cannot convert income ineligibility or a nonpositive allotment calculation into $2,196 annually."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $21,424 of taxable income and approximately $2,323 of tax, then submitted $940 without any computation supporting that value. Its final numeric output contradicts its own completed tax calculation."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated all Social Security benefits as AGI instead of including only $14,050.60 under the provisional-income formula. It also invented usable elderly and Saver’s Credits: retirement distributions are not qualified Saver’s Credit contributions, and the elderly credit is eliminated by the applicable income and nontaxable-Social-Security reductions."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model understated the aged-single standard deduction, using about $17,000 instead of $18,150 before adding the $6,000 senior deduction. The full $24,150 deduction produces taxable income of $21,423 rather than about $22,574."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at about $20,705. The statutory two-tier calculation includes $4,500 plus 85% of provisional income above $34,000, yielding $14,050.60, and it also failed to use the full $24,150 deduction."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly calculated approximately $21,424 of taxable income and approximately $2,323 of federal tax, then submitted $1,170. The submitted value has no basis in its reasoning and discards roughly half of its own computed liability."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the separate $6,000 senior deduction and therefore used only a roughly $17,000 aged standard deduction instead of total deductions of $24,150. It also deducted an unsupported $8 desired 401(k) contribution despite zero wages; the trace permits only a $0.40 above-the-line deduction."
-us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated tax of about $3,189 under its stated assumptions and then submitted $626 without applying any identified deduction or credit that produces that result. It also omitted the $6,000 senior deduction and improperly treated the desired $8 traditional 401(k) contribution as an AGI reduction despite no wages."
-us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $9,800 standard deduction and obsolete 10%/15% brackets, rather than the 2026 aged-single deduction of $18,150, the $6,000 senior deduction, and the applicable 10%/12% brackets. It also overstated taxable Social Security at $19,668.60 instead of $14,050.60."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a post-TCJA reversion to an $8,100 standard deduction, personal exemption, and 15% bracket. For 2026 the applicable deductions total $24,150, including the $6,000 senior deduction, and the second bracket applied here is 12%."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model’s stated $58,949 gross-taxable-income starting point treats the full Social Security benefit as taxable. Only $14,050.60 of Social Security enters AGI, and total deductions of $24,150 reduce taxable income to $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used projected pre-TCJA baseline deductions and a personal exemption instead of the operative 2026 deduction structure. It failed to apply the $18,150 aged-single standard deduction plus the separate $6,000 senior deduction."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied post-TCJA-expiration personal-exemption and 15% bracket assumptions and omitted the operative $6,000 senior deduction. It also reduced AGI by the desired $8 401(k) contribution despite the absence of wages, whereas the trace allows only $0.40 above the line."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly calculated taxable Social Security but subtracted only a $17,450 aged standard deduction. It omitted the separate $6,000 senior deduction and misstated the aged-single standard deduction, which together must total $24,150."
-us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted only $15,650 through an incorrect mixture of standard deduction, senior addition, and personal exemption. The applicable deduction is $18,150 for the aged-single standard deduction plus $6,000 for the senior deduction, totaling $24,150."
-us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the separate $6,000 senior deduction and used an approximate $17,000 aged standard deduction instead of $18,150. Total deductions are $24,150, producing $21,423 rather than $28,573.60 of taxable income."
-us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the absence of wages and dependents as eliminating federal tax, ignoring taxable retirement distributions, pension income, interest, and $14,050.60 of taxable Social Security. After the $24,150 deduction, $21,423 remains taxable and generates $2,322.76 before refundable credits."
-us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model asserted that deductions, contributions, and credits offset the income without calculating them. The deductible amounts total $24,150 plus only $0.40 above the line, no nonrefundable credit offsets the liability, and taxable income is $21,423."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $4,200 implies substantially more taxable income than the correct derivation. AGI of $45,573 is reduced by $24,150 of deductions to $21,423, and applying the 2026 main rates yields $2,322.76."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed a post-TCJA sunset regime with deductions and a personal exemption totaling $15,889 and then applied a 15% bracket. The operative deductions total $24,150, including the $6,000 senior deduction, and the income above the 10% bracket is taxed at 12%."
-us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model treated 85% of the entire Social Security benefit as taxable, but the two-tier provisional-income formula yields only $14,050.60. It also used an obsolete post-TCJA-reversion deduction and 15% bracket instead of $24,150 of deductions and the applicable 10%/12% rates."
-us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used the $18,150 aged-single standard deduction but omitted the additional $6,000 senior deduction. That omission raised taxable income from $21,423 to about $27,424 and overstated the tax."
-us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the required structured answer was missing."
-us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of the entire Social Security benefit and then declared tax near zero despite computing gross income above $61,000 and taxable income above $44,000 under its own assumptions. The correct Social Security inclusion is $14,050.60, and $21,423 remains taxable after all applicable deductions."
-us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model confused the 85% maximum inclusion rate with automatic inclusion of 85% of all benefits; the provisional-income formula yields $14,050.60, not $23,312.10. It also omitted the $6,000 senior deduction, and its final $8,587.50 contradicts its own recalculated tax near $4,500."
-us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model treated exceeding the upper provisional-income threshold as making 85% of all Social Security taxable, producing an internally impossible AGI, and omitted the $24,150 deduction structure. It then invented a $1,600 senior credit and a $2,100 survivor-with-dependent-children credit; the senior provision is a $6,000 deduction, and this filer has no dependent child."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $21,424 of taxable income and approximately $2,323 of tax, then submitted $940 instead. Its final value does not carry its own completed tax calculation into the output."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated all Social Security benefits as AGI and invented usable elderly and Saver’s Credits. At this income the elderly credit is eliminated under its statutory income limitations, the Saver’s Credit does not apply, and the tax after the applicable deductions remains $2,322.76."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an approximate $23,000 combined deduction instead of the exact $24,150 total consisting of the $18,150 aged standard deduction and $6,000 senior deduction. That overstated taxable income and produced $2,470 rather than applying the rates to $21,423."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model overstated taxable Social Security at about $20,705 instead of $14,050.60. It also used approximate deduction parameters rather than the exact $24,150 total, inflating taxable income to roughly $28,478 instead of $21,423."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly calculated taxable Social Security, the $24,150 deduction, taxable income near $21,424, and tax near $2,323, but submitted $1,170. The submitted value abandons its own completed computation without any supporting adjustment or credit."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted the separate $6,000 senior deduction and used only a projected $17,000 aged standard deduction. It also improperly treated the desired $8 traditional 401(k) contribution as reducing AGI despite no wages, while the trace permits only a $0.40 above-the-line deduction."
+us,scenario_085,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated tax of about $3,189 using an incomplete deduction and then submitted $626 without identifying any credit or valid computation producing that amount. It also omitted the $6,000 senior deduction that brings taxable income to $21,423."
+us,scenario_085,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used obsolete parameters: a $9,800 standard deduction and 10%/15% brackets with a $10,225 breakpoint. It also overstated taxable Social Security at $19,668.60; the correct inclusion is $14,050.60 and the applicable deductions total $24,150."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model assumed a TCJA expiration regime with an $8,100 standard deduction, personal exemption, and 15% bracket. The applicable 2026 computation instead uses the $18,150 aged standard deduction plus the $6,000 senior deduction and current 10%/12% rates."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated $58,949 as gross taxable income, thereby failing to limit taxable Social Security to $14,050.60. It also failed to apply the exact $24,150 combined standard and senior deductions that yield $21,423 of taxable income."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Although it derived the correct AGI, the model applied projected pre-TCJA standard-deduction and personal-exemption rules. It omitted the applicable $18,150 aged standard deduction plus $6,000 senior deduction, which reduce taxable income to $21,423."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used a post-TCJA-expiration deduction-and-exemption regime and therefore left about $30,716 taxable. It should have deducted $24,150 from AGI after the $0.40 adjustment, producing taxable income of $21,423 and taxation under the 10%/12% schedule."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly computed taxable Social Security and AGI but deducted only a $17,450 aged standard deduction. It omitted the separate $6,000 senior deduction and did not use the exact $18,150 standard deduction, leaving taxable income overstated."
+us,scenario_085,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model subtracted only $15,650 through an inapplicable mix of standard deduction, senior addition, and personal exemption. The correct taxable-income deductions total $24,150, with no personal-exemption substitution for the $6,000 senior deduction."
+us,scenario_085,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI and taxable Social Security but used only an approximate $17,000 aged standard deduction. It omitted the separate $6,000 senior deduction, so it taxed $28,573.60 rather than $21,423."
+us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model assumed retirement income and Social Security produced no taxable liability without computing taxable Social Security, AGI, or taxable income. Taxable distributions, pension, interest, and $14,050.60 of Social Security produce $21,423 of taxable income after deductions."
+us,scenario_085,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions and retirement-related credits offset the income completely. After the exact $24,150 deductions, $21,423 remains taxable, and no nonrefundable credit eliminates the resulting $2,322.76 tax."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted $4,200 implies substantially more taxable income than the correct $21,423. The model failed to apply the full $24,150 deduction package after including only $14,050.60 of Social Security in AGI."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model assumed TCJA sunset rules, substituted a personal exemption, and used only $15,889 of combined reductions. It omitted the applicable $6,000 senior deduction and used a 15% second bracket instead of the 2026 12% rate."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model applied a reverted $8,550 standard deduction, a personal exemption, and a 15% bracket. The applicable computation uses an $18,150 aged standard deduction, an additional $6,000 senior deduction, and 10%/12% main rates."
+us,scenario_085,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $23,264 of Social Security, effectively treating the 85% maximum as the taxable amount rather than applying the two-tier worksheet that yields $14,050.60. It then used a post-TCJA-reversion $10,631 deduction and 15% rate instead of the $24,150 deductions and applicable 10%/12% schedule."
+us,scenario_085,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model correctly found AGI near $45,574 but applied only the $18,150 aged standard deduction. It omitted the separate $6,000 senior deduction, overstating taxable income by $6,000 and tax by $720."
+us,scenario_085,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. The required structured answer was therefore missing.
+us,scenario_085,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated 85% of Social Security as automatically taxable instead of applying the worksheet, yet then asserted that roughly $44,599 of taxable income after its own deduction produced near-zero tax. Both its Social Security inclusion and its conclusion that the standard deduction eliminates liability are arithmetically wrong."
+us,scenario_085,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model omitted the $4,500 second-tier base amount when calculating taxable Social Security, using only 85% of provisional income above $34,000. Correct taxable Social Security is $14,050.60, not $9,550.60; its stated $1,782.83 calculation also does not support the submitted $3,680."
+us,scenario_085,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model incorrectly used the full 85% maximum of Social Security instead of the worksheet amount of $14,050.60 and omitted the $6,000 senior deduction. Even its resulting reasoning calculated tax near $4,500, but it submitted $8,587.50 without any computation supporting the extra liability."
+us,scenario_085,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model treated exceeding $34,000 of provisional income as requiring inclusion of the full 85% of Social Security, rather than applying the worksheet amount of $14,050.60. It also invented a $1,600 Senior Bonus credit and a $2,100 survivor-with-dependent-children credit even though the senior provision is a deduction and no dependent child exists."
 us,scenario_085,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_085,local_income_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_085,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model correctly stated that the household had no wages and computed zero employee Social Security, Medicare, and state payroll taxes, but then submitted 1459 instead of carrying that zero into the value field. The submitted number is unrelated to its stated derivation."
@@ -5186,44 +5447,47 @@ us,scenario_086,child1_medicaid_eligible,claude-sonnet-4.6,llm_error,taxable_inc
 us,scenario_086,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model invented a three-person household despite the prompt listing only the head and child, then conflated the 247% FPL PeachCare/CHIP ceiling with Medicaid eligibility. For the actual two-person household, child1 is at 3.44 times FPL and has no Medicaid eligibility category."
 us,scenario_086,child1_medicaid_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model relied on age-based categorical eligibility without applying Georgia’s income threshold for a 12-year-old child. Child1’s MAGI income level is 3.44 times FPL, above the applicable child Medicaid limits, leaving the Medicaid category as NONE."
 us,scenario_086,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no answer or explanation for child1_medicaid_eligible, violating the required output contract."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable portion of the dependent Social Security benefits and improperly deducted $275 of auto-loan interest. It also submitted $3,305 despite its own calculation ending at $1,925; the required taxable income is $48,412.90 and the tax after the $2,200 credit is $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model treated the head as single instead of head of household and incorrectly deducted employer and other health-insurance premiums from AGI. It also used a $2,000 rather than $2,200 child tax credit and submitted a figure inconsistent with every calculation in its explanation."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded all $10,900.40 of taxable Social Security benefits, understated the educator deduction, and used an estimated $23,000 standard deduction rather than $24,150. It then used a $2,000 child credit instead of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount and used estimated deduction and bracket parameters. Its submitted $4,673 is the estimated tentative tax it discussed rather than tax after the nonrefundable child credit, so it also failed to subtract the credit in the submitted output."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model excluded the taxable Social Security benefits and consequently taxed only about $37,162 rather than $48,412.90. It also used a $2,000 child credit instead of the applicable $2,200 credit and replaced the exact 2026 parameters with estimates."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA rules expired for this 2026 calculation and reconstructed pre-TCJA standard deductions, personal exemptions, 15% brackets, and a $1,000 child credit. The applicable computation instead uses the $24,150 head-of-household standard deduction, current 2026 brackets, and a $2,200 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the $10,900.40 taxable Social Security amount as nontaxable, producing taxable income far below $48,412.90. It also used estimated thresholds and a $2,000 credit, then submitted $3,892 despite its own arithmetic yielding $2,271."
-us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied an obsolete pre-TCJA structure with an $11,000 standard deduction, personal exemptions, 15% rates, and a $1,000 child credit. It also deducted $1,560 of insurance premiums from wages and omitted the $10,900.40 taxable Social Security amount."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model deducted the $19,648 employer-sponsored premium from wages even though the stated $62,000 wage input is the taxable wage amount used by the engine. It then applied erroneous post-TCJA-expiration personal exemptions, a $16,500 standard deduction, 15% rates, and only a $1,000 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used the full $12,824 Social Security benefit in gross income rather than its $10,900.40 taxable portion and used a $15,300 deduction rather than the $24,150 head-of-household standard deduction. It also used a $2,000 child credit rather than $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly restored personal exemptions and a $1,000 child credit under assumed TCJA-expiration rules. The applicable 2026 calculation has no personal-exemption subtraction, uses the $24,150 head-of-household standard deduction, and subtracts a $2,200 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted the employer-sponsored premium as a $19,648 reduction of the stated wage input and introduced personal exemptions that do not apply. It also used a $15,000 standard deduction, capped the educator deduction at $300, and allowed only a $1,000 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The unexplained calculation does not reproduce the required chain of $72,900.40 gross income, $337.50 educator deduction, $24,150 standard deduction, $5,455.55 tentative tax, and $2,200 credit. Its $3,678 answer therefore reflects an incorrect underlying tax-base or bracket computation despite claiming the correct filing status and credit category."
-us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model explicitly subtracted personal exemptions or other unspecified deductions that are absent from the applicable calculation. Taxable income is $48,412.90 after only the $337.50 adjustment and $24,150 standard deduction, and the resulting tax after the $2,200 credit is $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount and used an estimated $22,500 standard deduction. It also capped the educator deduction at $300 and used a $2,000 rather than $2,200 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly concluded that nonrefundable credits fully offset the tentative tax. The 2026 bracket calculation produces $5,455.55 before credits, and the available $2,200 child credit leaves $3,255.55 rather than zero."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized medical deductions even though the household takes the $24,150 standard deduction and the listed medical costs do not separately reduce taxable income. It also failed to identify and apply the $2,200 nonrefundable child credit in the required computation."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model excluded $10,900.40 of taxable Social Security benefits and used an estimated $22,500 standard deduction. It then treated only $384 of the $2,200 child credit as nonrefundable, whereas the $5,455.55 tentative liability permits the entire $2,200 credit to offset tax."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included only $6,412 of Social Security benefits, equal to one-half of the gross benefit, instead of the computed taxable amount of $10,900.40. That understated AGI and taxable income before applying the otherwise applicable $2,200 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the correct categories but its $3,269 result uses imprecise 2026 parameter values or arithmetic. With taxable income of $48,412.90, exact 2026 rates produce $5,455.55 before the $2,200 credit and $3,255.55 afterward."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model introduced a mortgage-interest deduction even though only the mortgage balance was supplied and the household uses the $24,150 standard deduction. Its calculation also failed to follow the traced inclusion of $10,900.40 in taxable Social Security benefits and the resulting $48,412.90 taxable income."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied assumed post-TCJA-expiration rules, including a $12,389 standard deduction, personal exemptions, 15% brackets, and a $1,000 child credit. The applicable 2026 rules instead yield a $24,150 standard deduction, no personal exemptions, and a $2,200 credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly restored pre-TCJA personal exemptions, brackets, and a $1,000 child credit while using a $12,538 standard deduction. The applicable 2026 head-of-household parameters produce $48,412.90 of taxable income and allow a $2,200 credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model rounded the taxable Social Security amount, capped the educator deduction at $300 rather than $337.50, and rounded the resulting tax. Exact inputs produce AGI of $72,562.90, taxable income of $48,412.90, and tax after the $2,200 credit of $3,255.55."
-us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric output or explanation for the requested variable, so the required structured answer could not be parsed."
-us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $275 of auto-loan interest from AGI. Removing that unsupported deduction raises taxable income from $48,137.40 to $48,412.90 and yields $3,255.55 after the $2,200 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model used single-filer deductions and brackets despite the head qualifying as head of household. It also excluded $10,900.40 of taxable Social Security benefits and used a $2,000 rather than $2,200 child credit."
-us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the head and child as married filing jointly, even though a child is not a spouse and the correct status is head of household. That caused it to use the MFJ Social Security thresholds, standard deduction, and tax brackets; it also capped the educator deduction at $300 and used a $2,000 credit instead of $2,200."
-us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used an unsupported $32,200 standard deduction and excluded the taxable portion of the Social Security benefits, reducing taxable income to zero. Under head-of-household rules, taxable income is $48,412.90 and the $2,200 child credit does not eliminate the $5,455.55 tentative tax."
-us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model treated the $1,700 refundable CTC cap as an automatic refundable amount even though it calculated roughly $4,300 of tax liability. That liability absorbs the entire available CTC as a nonrefundable credit, leaving refundable CTC of $0."
-us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model applied the $1,700 ACTC cap without first allocating the Child Tax Credit against the filer’s tax liability. Because the liability can absorb the full CTC, no unused credit remains refundable."
-us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly stated that the roughly $4,300 tax liability absorbs the full CTC and leaves no ACTC, then contradicted that computation by inventing a $1,700 refundable remainder. No listed deduction reduces liability enough to create that remainder."
-us,scenario_086,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model inferred a $1,000 ACTC solely from the presence of a qualifying child. The filer’s tax liability uses the full Child Tax Credit nonrefundably, so the refundable remainder is $0."
-us,scenario_086,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model described $62,000 of wages as “limited earnings” and assigned an approximate refundable CTC without computing the credit’s allocation against tax. The resulting liability absorbs the full CTC nonrefundably, leaving no refundable portion."
-us,scenario_086,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model invented a $384 CTC remainder after the nonrefundable portion without calculating the actual tax liability. The liability is sufficient to use the entire available CTC, so the residual refundable amount is $0."
-us,scenario_086,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model treated the $1,700 per-child refundable ceiling as an entitlement whenever a qualifying child exists. The ceiling does not create a refundable credit when the full CTC is used against tax liability."
-us,scenario_086,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required submission contract."
-us,scenario_086,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly based refundability on the child’s separate tax liability and treated the refundable cap as automatic. The CTC belongs to the head’s tax unit, whose liability absorbs the entire credit nonrefundably; the child’s dependent benefits do not make the credit fully refundable."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded $10,900.40 of taxable Social Security benefits and improperly deducted $275 of auto-loan interest, instead of deriving $48,412.90 of taxable income. Its submitted $3,305 also contradicts its own stated final calculation of $1,925."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status and a $14,600 deduction instead of head-of-household status and the $24,150 standard deduction. It also deducted employer and other insurance premiums from AGI, included the full Social Security benefit rather than $10,900.40, and used a $2,000 rather than $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable portion of Social Security benefits, leaving taxable income near $38,700 instead of $48,412.90. It also used estimated deduction and bracket parameters and a $2,000 child credit rather than the applicable $2,200 credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded $10,900.40 of taxable Social Security benefits and used estimated deduction, bracket, and child-credit parameters. Its submitted $4,673 is the tentative-tax estimate it stated before credits, even though the requested output subtracts the nonrefundable child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated the dependent Social Security benefits as nontaxable and therefore omitted $10,900.40 from gross income. It also used estimated 2026 parameters and a $2,000 child credit instead of calculating $5,455.55 of tentative tax less $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly imposed assumed post-TCJA-sunset rules, including personal exemptions, a small standard deduction, pre-TCJA brackets, and a $1,000 child credit. The applicable 2026 computation instead uses the $24,150 head-of-household deduction, current brackets, no personal exemptions, and a $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount, producing taxable income of only about $38,350 rather than $48,412.90. Its submitted $3,892 also does not follow its own stated tentative tax of $4,271 less a $2,000 credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used invented pre-TCJA-style personal exemptions, a $11,000 standard deduction, 15% brackets, and a $1,000 child credit. It also deducted $1,560 of health premiums from wages while omitting the $10,900.40 taxable Social Security amount."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted $19,648 of employer-sponsored premiums from the stated gross wages even though those wages are the tax input, reducing wage income to $42,352. It compounded that error with post-TCJA-sunset personal exemptions, the wrong deduction and rates, and a $1,000 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used the full $12,824 Social Security benefit rather than the taxable $10,900.40 and applied a $15,300 deduction instead of the $24,150 head-of-household deduction. Its unexplained arithmetic also fails to reproduce the applicable $48,412.90 taxable income and $2,200 nonrefundable child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied assumed post-TCJA-sunset personal exemptions, pre-TCJA tax parameters, and a $1,000 child credit. The applicable computation has taxable income of $48,412.90 under the $24,150 head-of-household standard deduction and subtracts a $2,200 credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated $62,000 wages by $19,648 of employer-sponsored premiums. It also applied personal exemptions, the wrong standard deduction and rates, and only a $1,000 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The submitted amount implies a tentative-tax or taxable-income calculation different from the traced $5,455.55 tentative tax on $48,412.90. Although it recognized a child credit, it used $2,000 rather than the applicable $2,200 nonrefundable credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model invoked personal exemptions even though the applicable 2026 calculation uses no personal exemptions. Its answer also implies incorrect deduction, bracket, or child-credit parameters rather than $5,455.55 of tentative tax less $2,200."
+us,scenario_086,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model excluded the $10,900.40 taxable Social Security amount, yielding $39,200 rather than $48,412.90 of taxable income. It also used estimated deduction and bracket values and a $2,000 rather than $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model incorrectly treated nonrefundable credits as sufficient to eliminate all liability. Tentative tax is $5,455.55 and the available nonrefundable child credit is $2,200, leaving $3,255.55 rather than zero."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated listed medical expenses as itemized deductions without establishing that itemizing exceeds the $24,150 standard deduction. It also failed to apply the traced taxable Social Security inclusion and the $2,200 nonrefundable child credit needed to reach $3,255.55."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model excluded $10,900.40 of taxable Social Security benefits and therefore understated taxable income. It then treated only $384 of the $2,200 child credit as reducing pre-refund tax, whereas the full $2,200 is usable nonrefundably against $5,455.55 of tentative tax."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model included only $6,412 of Social Security benefits, exactly half of the benefit, instead of the taxable amount of $10,900.40. That understates AGI and taxable income before applying the otherwise recognized $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the correct components but its $3,269 result does not apply the exact 2026 head-of-household brackets to $48,412.90. That calculation produces $5,455.55 before the $2,200 child credit and $3,255.55 afterward."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated unspecified mortgage interest as a deduction even though only the mortgage balance was supplied and unlisted amounts are zero. Its answer also fails to use the standard-deduction computation yielding $48,412.90 of taxable income and then subtract the $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied assumed post-TCJA-sunset personal exemptions, a $12,389 standard deduction, 15% brackets, and a $1,000 child credit. The applicable rules use a $24,150 head-of-household standard deduction, current 2026 brackets, and a $2,200 credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly restored personal exemptions, a small post-sunset standard deduction, 15% brackets, and a $1,000 child credit. It also capped the educator deduction at $300 instead of applying the traced $337.50 adjustment."
+us,scenario_086,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly used assumed post-TCJA-sunset personal exemptions, a $12,538 standard deduction, 15% brackets, and a $1,000 child credit. The traced 2026 rules instead yield $48,412.90 of taxable income and subtract a $2,200 credit from $5,455.55."
+us,scenario_086,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model closely reproduced the correct pathway but rounded the taxable Social Security amount, capped the educator adjustment at $300 instead of $337.50, and approximated the brackets. Exact inputs produce $48,412.90 of taxable income, $5,455.55 of tentative tax, and $3,255.55 after the $2,200 credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable. This is a missing-output contract failure rather than a substantive tax calculation.
+us,scenario_086,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $275 of auto-loan interest from AGI. Removing that unauthorized deduction raises taxable income from $48,137.40 to $48,412.90 and produces the traced $3,255.55 result after the child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model calculated the tax as single, using a $15,000 deduction and single brackets instead of head-of-household status with a $24,150 deduction. It also excluded $10,900.40 of taxable Social Security benefits and used a $2,000 rather than $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model used single filing status instead of head-of-household status and improperly deducted $275 of auto-loan interest. It also treated only $500 of the $2,200 child credit as nonrefundable, even though the full credit is usable against tentative tax."
+us,scenario_086,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the head and child as married filing jointly, but a child is not a spouse and the correct filing status is head of household. Its $31,200 joint deduction and joint brackets therefore replaced the applicable $24,150 head-of-household deduction and brackets; it also used a $2,000 rather than $2,200 child credit."
+us,scenario_086,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model claimed that a $32,200 deduction and exclusion of the dependent benefits reduced taxable income to zero, an arithmetic impossibility even under its own $74,824 gross-income premise. The correct computation includes $10,900.40 of taxable Social Security benefits, produces $48,412.90 of taxable income, and leaves $3,255.55 after the child credit."
+us,scenario_086,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model reversed the CTC ordering when it said the nonrefundable portion absorbs tax while leaving the $1,700 refundable cap. The household’s pre-credit liability exceeds the available CTC, so the entire credit is used nonrefundably and no ACTC remains."
+us,scenario_086,federal_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model treated the $1,700 ACTC cap as an automatic refundable payment despite recognizing substantial federal tax liability. Because that liability absorbs the full CTC as a nonrefundable credit, the refundable remainder is zero."
+us,scenario_086,federal_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model correctly calculated that roughly $4,300 of tax liability would absorb the full CTC, then contradicted that calculation by inventing unspecified deductions and assigning a $1,700 refundable remainder. Unlisted deductions are zero, and the stated household facts leave enough liability to use the CTC entirely nonrefundably."
+us,scenario_086,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model assumed that having a qualifying child alone produces an ACTC. The household’s federal tax liability absorbs the entire available CTC nonrefundably, leaving no refundable portion."
+us,scenario_086,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model characterized $62,000 of wages as limited earnings and inferred a positive ACTC solely from the qualifying child. Earnings satisfy the ACTC phase-in, but refundability is also limited to CTC left after application against tax liability; here none remains."
+us,scenario_086,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model understated how much CTC the household’s federal tax liability can absorb and assigned a $384 refundable remainder. The liability is sufficient to use the full child credit nonrefundably, so the remainder is zero."
+us,scenario_086,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model treated the $1,700 refundable-child-credit ceiling as the amount payable whenever the earnings phase-in is satisfied. The ceiling does not override the requirement that ACTC be limited to the CTC unused against federal tax liability, and this household has no unused CTC."
+us,scenario_086,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract."
+us,scenario_086,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model checked the ACTC earnings phase-in and MAGI phaseout but omitted the limitation based on CTC remaining after nonrefundable use. The household’s tax liability consumes the full CTC, so satisfying the phase-in and remaining below the MAGI phaseout does not generate a refund."
+us,scenario_086,federal_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly based CTC refundability on the child’s separate tax liability and treated the dependent benefits as making the credit fully refundable. The CTC belongs to the head’s tax unit and is first applied against that unit’s liability, which absorbs it in full; the child is also a qualifying child for EITC, though income eliminates the EITC."
 us,scenario_086,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_086,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_086,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model correctly calculated $4,743 of federal FICA, then improperly added $466.80 of Georgia unemployment insurance as an employee payroll tax. Georgia unemployment insurance is an employer liability and is excluded from this employee-side output."
@@ -5234,45 +5498,48 @@ us,scenario_086,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"T
 us,scenario_086,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model stated the correct component amounts—$3,844 of Social Security tax and $899 of Medicare tax—but failed to add them correctly. Those components total $4,743, not $3,979, and no Georgia-specific adjustment changes that sum."
 us,scenario_086,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required structured-output contract."
 us,scenario_086,payroll_tax,qwen3.8-max,llm_error,other,False,"The model stated the correct $3,844 Social Security tax and $899 Medicare tax but submitted $3,813 instead of adding them. The stated components total $4,743."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used an invented $24,000 head-of-household standard deduction and a $4,000 dependent exemption, then submitted a number it explicitly identified as single-filer treatment instead of its own calculation. Georgia applies a $15,000 standard deduction and $5,000 dependent exemption here, leaving $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied Georgia's obsolete graduated 1%–6% brackets to unreduced wage income and treated the filer as single. It omitted the Social Security subtraction, $15,000 head-of-household standard deduction, and $5,000 dependent exemption that reduce taxable income to $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model replaced the applicable $15,000 standard deduction and $5,000 dependent exemption with $18,500 and $4,000. Its submitted $2,191 also does not follow from its stated $39,162 taxable income and 5.19% multiplication."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated the head-of-household filer as single, using a $12,000 standard deduction and $4,000 dependent exemption instead of $15,000 and $5,000. Its final $1,869 is also disconnected from its stated $2,324 calculation."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $12,824 Social Security benefit from an AGI that already excluded it, rather than starting with $72,562.90 and subtracting only the $10,900.40 taxable portion. It also used obsolete single-filer deductions and exemptions instead of the $15,000 head-of-household standard deduction and $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the filer as single and used a $12,000 standard deduction plus a $4,000 dependent exemption instead of $15,000 and $5,000. Its submitted $2,765 further contradicts its own stated estimate of about $2,372."
-us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $60,140 AGI and applied obsolete $5,400 standard-deduction and personal-exemption amounts. The required sequence yields Georgia AGI of $61,662.50 and taxable income of $41,662.50 after the $15,000 head-of-household deduction and $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model started from an unsupported federal AGI of $52,952.40 and treated a $15,800 personal exemption as the principal head-of-household deduction. The computation instead reaches Georgia AGI of $61,662.50 and subtracts a $15,000 standard deduction plus a $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an overstated taxable base from applying generic deductions to the $62,000 wage amount. The required Social Security adjustment and Georgia deductions produce taxable income of $41,662.50, not the base consistent with $2,690 of tax."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly reached Georgia AGI near $61,662 but used a $12,000 standard deduction and $3,000 dependent deduction. The applicable amounts are $15,000 and $5,000, reducing taxable income by an additional $5,000 to $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported federal AGI of $52,952 and derived taxable income of about $37,952. The trace instead starts from $72,562.90, subtracts $10,900.40 of taxable Social Security, and applies $20,000 of Georgia deductions and exemptions to reach $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The $3,047 answer implies that the model taxed substantially more than the correct $41,662.50 taxable income. It failed to apply the full sequence of the $10,900.40 Social Security subtraction, $15,000 head-of-household standard deduction, and $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $1,432.32 answer implies excessive deductions or an understated taxable base. The specified Georgia adjustments leave $41,662.50 taxable after exactly a $15,000 standard deduction and $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly approximated Georgia AGI but used only a $10,600 head-of-household standard deduction and omitted the separate $5,000 dependent exemption. Those two allowances total $20,000 and reduce taxable income to $41,662.50."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The $1,639 answer implies an excessive reduction of Georgia taxable income beyond the permitted adjustments. Georgia AGI is $61,662.50, and the $15,000 standard deduction plus $5,000 dependent exemption leave $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly imported federal-style medical adjustments into the Georgia taxable-income calculation and reduced the base to a level consistent with only $498 of tax. The applicable state computation leaves $41,662.50 taxable after the Social Security subtraction, standard deduction, and dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model bypassed the traced federal AGI and Social Security subtraction and used a $12,000 standard deduction with only $4,500 of exemptions. Georgia instead applies $15,000 and $5,000 to Georgia AGI of $61,662.50, producing $41,662.50 taxable income."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $1,357.80 result implies that the model over-deducted income after removing taxable Social Security. The applicable head-of-household standard deduction and dependent exemption total $20,000, leaving $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $12,000 and $4,000 allowances instead of the $15,000 head-of-household standard deduction and $5,000 dependent exemption. It therefore overstated taxable income as $45,662.40 rather than $41,662.50 and also applied the wrong rate treatment."
-us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $1,903 result reflects an understated taxable base or rate after the Social Security subtraction. Applying the specified $15,000 standard deduction and $5,000 dependent exemption to $61,662.50 yields $41,662.50 taxable and $2,078.96 of tax."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The terse $2,184 result does not apply the rate schedule correctly to the traced taxable income. The Georgia adjustments produce $41,662.50 taxable, on which the applicable schedule yields $2,078.96."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model collapsed the filing-status deduction and dependent exemption into an unsupported $18,000 deduction. The applicable amounts are a $15,000 standard deduction and a separate $5,000 dependent exemption, leaving $41,662.50 rather than $43,662 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the educator expense after beginning with federal AGI and used obsolete $5,400 deduction and exemption amounts. The correct state allowances are a $15,000 head-of-household standard deduction and $5,000 dependent exemption, producing $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model approximated taxable income as $40,700 instead of carrying through the traced amounts. Georgia AGI of $61,662.50 less the $15,000 standard deduction and $5,000 dependent exemption equals $41,662.50, which yields $2,078.96 under the applicable schedule."
-us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used a $12,000 standard deduction and two $2,700 personal exemptions instead of the $15,000 head-of-household standard deduction and $5,000 dependent exemption. It also applied an incorrect 4.99% rate, compounding its overstated $44,262 taxable income."
-us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model misstated Georgia AGI as $61,387 and used an $18,500 standard deduction plus a $3,000 dependent exemption. The traced amounts are $61,662.50 of Georgia AGI, a $15,000 standard deduction, and a $5,000 dependent exemption, leaving $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $5,000 standard deduction and a $2,700 personal exemption while omitting the $5,000 dependent exemption and the traced Social Security adjustment. The applicable $15,000 head-of-household standard deduction and $5,000 dependent exemption reduce Georgia AGI to $41,662.50 taxable."
-us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model taxed the full $12,824 Social Security benefit by starting from $74,824 rather than subtracting the $10,900.40 taxable portion from federal AGI. It also used a $12,000 standard deduction and obsolete personal/dependent exemptions instead of the applicable $15,000 standard deduction and $5,000 dependent exemption."
-us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model's arithmetic is internally inconsistent: excluding Social Security from $74,824 and then subtracting $23,100 cannot yield its stated $51,700 taxable income. The traced computation produces Georgia AGI of $61,662.50 and taxable income of $41,662.50 after the $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model replaced the applicable $15,000 head-of-household standard deduction and $5,000 dependent exemption with unsupported $24,000 and $4,000 amounts. Its submitted $2,732 also contradicts its own recomputed $1,749 figure, so it never carried a coherent taxable-income calculation into the output."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model applied Georgia's obsolete graduated 1%–6% bracket schedule directly to $62,000. It omitted the federal-AGI starting point, Georgia's Social Security subtraction, and the $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an $18,500 standard deduction and $4,000 dependent exemption instead of $15,000 and $5,000, and it omitted taxable Social Security when constructing federal AGI before the Georgia subtraction. Its final $2,191 also does not follow from its stated $39,162 taxable income and 5.19% calculation."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used a $12,000 standard deduction and $4,000 dependent exemption rather than $15,000 and $5,000, then applied the wrong rate. Its submitted $1,869 is also disconnected from its own calculated $2,324 liability."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model subtracted the entire $12,824 Social Security benefit from an AGI that already excluded it, double-counting the exclusion. It then used obsolete $5,400, $2,700, and $3,000 deduction and exemption amounts instead of the applicable $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from federal AGI and used $12,000 and $4,000 rather than the applicable $15,000 standard deduction and $5,000 dependent exemption. Its $2,765 output also does not result from its stated $45,700 taxable income and 5.19% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model constructed the wrong $60,140 Georgia AGI and used obsolete $5,400 standard-deduction and $5,400 personal-exemption amounts. The correct base is $61,662.50, reduced by $15,000 and $5,000 to $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model began from an unsupported $52,952.40 federal AGI instead of $72,562.90 and therefore did not execute the taxable-Social-Security inclusion followed by Georgia's $10,900.40 subtraction. It also substituted a $15,800 personal exemption and $3,000 dependent exemption for the applicable $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a Georgia taxable-income base substantially above the correct $41,662.50. It failed to combine Georgia AGI of $61,662.50 with the full $15,000 standard deduction and $5,000 dependent exemption before applying the 2026 tax rate."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly reached approximately $61,662 of Georgia AGI but deducted only $12,000 and $3,000. The applicable $15,000 standard deduction and $5,000 dependent exemption reduce taxable income to $41,662.50, not $46,662."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $52,952 federal AGI and an unexplained $37,952 taxable-income figure. The required calculation starts with $72,562.90 of federal AGI, subtracts $10,900.40 of taxable Social Security, and then deducts $20,000 to reach $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted $3,047 implies that the model retained far more income than the correct $41,662.50 taxable base. It failed to apply the traced $10,900.40 Social Security subtraction together with the $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The submitted $1,432.32 implies a taxable base far below $41,662.50. The model over-deducted income instead of stopping after the $10,900.40 Georgia subtraction, $15,000 standard deduction, and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"Although the model reached roughly the correct Georgia AGI, it allowed only a $10,600 standard deduction and omitted the $5,000 dependent exemption. It therefore taxed $51,062 instead of $41,662.50 and also used the wrong rate."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $1,639 implies taxable income materially below the traced $41,662.50. The model's generic reference to deductions did not preserve the exact $61,662.50 Georgia AGI less the $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model imported federal-style and medical adjustments that do not produce the benchmark's Georgia base, reducing taxable income drastically below $41,662.50. The correct computation uses the $10,900.40 Georgia Social Security subtraction and exactly $20,000 of standard and dependent deductions."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from federal AGI and used a $12,000 standard deduction plus only $4,500 of exemptions. Georgia AGI is $61,662.50 and the applicable deductions are $15,000 and $5,000, producing $41,662.50 before the applicable rate."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The $1,357.80 result implies that the model over-applied Georgia deductions or excluded more Social Security than the taxable $10,900.40 subtraction. The correct taxable income after the stated subtraction, standard deduction, and dependent exemption is $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used $12,000 and $4,000 deductions, leaving $45,662.40 instead of applying the $15,000 standard deduction and $5,000 dependent exemption to reach $41,662.50. It also applied an incorrect 5.09% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $1,903 output does not follow from the correct $41,662.50 taxable-income base. The model's unspecified head-of-household deductions and exemptions over-reduced the base and it also used the wrong 5.19% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $2,184 answer implies a taxable base or rate above the traced result. The model failed to preserve the exact $41,662.50 obtained from $61,662.50 of Georgia AGI less $15,000 and $5,000."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model combined the standard deduction and dependent exemption into an incorrect $18,000 reduction. The applicable reductions total $20,000, leaving $41,662.50 rather than $43,662, and the applicable rate is not 5.29%."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $12,000 standard deduction and $5,700 of exemptions rather than $15,000 and $5,000. It consequently left about $44,000 taxable instead of $41,662.50 and compounded the error with a 5.39% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted the $338 educator expense a second time after starting from federal AGI and used obsolete $5,400 deduction and $5,400 exemption amounts. The correct deductions from $61,662.50 of Georgia AGI are $15,000 and $5,000, and the model also used the wrong 5.49% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model approximated taxable income at $40,700 instead of the traced $41,662.50 and applied an incorrect 5.29% rate. Applying the applicable rate to the exact taxable base produces $2,078.96."
+us,scenario_086,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from federal AGI, then used a $12,000 standard deduction and two $2,700 exemptions. The correct Georgia AGI is $61,662.50 and the correct reductions are $15,000 plus a single $5,000 dependent exemption, yielding $41,662.50."
+us,scenario_086,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated Georgia AGI by $275 and used an $18,500 standard deduction plus a $3,000 dependent exemption. The correct amounts are $61,662.50 of Georgia AGI, a $15,000 standard deduction, and a $5,000 dependent exemption; it also used the wrong 5.19% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model started from wages rather than federal AGI after taxable Social Security and allowed only a $5,000 standard deduction and $2,700 personal exemption. It omitted the traced $10,900.40 Georgia subtraction and the applicable $15,000 standard deduction and $5,000 dependent exemption."
+us,scenario_086,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model understated federal and Georgia AGI by $275 and substituted a $12,000 standard deduction, $3,700 personal exemption, and $4,000 dependent exemption for the applicable $15,000 and $5,000 deductions. It also applied an incorrect 5.09% rate."
+us,scenario_086,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated the full $12,824 Social Security benefit as Georgia taxable income instead of including only $10,900.40 in federal AGI and then subtracting that taxable amount for Georgia. It also used incorrect $12,000, $2,700, and $3,000 deductions and a 5.12% rate rather than reducing $61,662.50 by $15,000 and $5,000."
+us,scenario_086,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model claimed Social Security was excluded but nevertheless left $51,700 taxable, which is incompatible with Georgia AGI of $61,662.50 and the applicable $20,000 total deductions. The correct taxable income is $41,662.50, not $51,700."
 us,scenario_086,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Georgia refundable dependent-child credit of $260 per qualifying child. Georgia has no such applicable refundable credit, so the 12-year-old dependent generates $0 in state refundable credits."
 us,scenario_088,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted the deduction for the $9,600 of qualified tips included in wages, leaving $7,085 of purported taxable income. It also incorrectly included the $675 state tax refund in gross income without establishing a prior-year itemized-deduction tax benefit."
 us,scenario_088,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither a numeric value nor an explanation for the requested output, violating the required output contract."
 us,scenario_088,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model's reasoning arrived at approximately zero tax but submitted -$3. Federal income tax before refundable credits is floored at zero, and the qualified-tip deduction reduces taxable income sufficiently to produce exactly $0; it also incorrectly described an age-based additional standard deduction at age 56, when that addition begins at age 65."
 us,scenario_088,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly excluded the state tax refund and reached AGI of about $23,360, but it omitted the deduction for the $9,600 of qualified tips included in wages. Applying that deduction alongside the standard deduction eliminates the $6,810 of taxable income on which it computed $681 of tax."
-us,scenario_088,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using $18,577 of wages instead of the greater applicable amount, the household's approximately $24,035 AGI. That AGI fully phases out the credit; the model also contradicted its own phaseout arithmetic by jumping from an estimated $76 credit to $632."
-us,scenario_088,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or reasoning for federal_refundable_credits, violating the required structured-output contract."
-us,scenario_088,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model calculated the childless EITC phaseout solely from wages of $18,577. The phaseout instead uses the greater of earned income and AGI, and the approximately $24,035 AGI reduces the EITC to $0."
-us,scenario_088,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated gross earned income below the wage-based limit as sufficient for a childless EITC and omitted the AGI phaseout test. Approximately $24,035 of AGI exceeds the single no-child limit and yields no EITC."
-us,scenario_088,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model compared only $18,577 of earned income with its estimated childless EITC cutoff and ignored the rule that the phaseout uses the greater of earned income and AGI. The household's approximately $24,035 AGI fully phases out the EITC, leaving federal refundable credits at $0."
+us,scenario_088,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout to $18,577 of wages instead of the greater phaseout base of earned income or approximately $24,035 AGI. That AGI fully phases out the no-child EITC; its submitted $632 also contradicts its own phaseout arithmetic, which calculated only about $76 before the AGI correction."
+us,scenario_088,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable output or reasoning for federal_refundable_credits, violating the required submission contract."
+us,scenario_088,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model subtracted the phaseout using only $18,577 of earned income. The no-child EITC phaseout uses the greater of earned income or AGI, and approximately $24,035 AGI reduces the credit to $0."
+us,scenario_088,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated gross earned income below the nominal no-child EITC limit as sufficient for a positive credit and omitted the AGI-based phaseout test. Approximately $24,035 AGI exceeds the phaseout endpoint, so the EITC is $0."
+us,scenario_088,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model phased out the childless EITC against $18,577 of earnings alone. The statutory phaseout base is the greater of earned income or AGI, and approximately $24,035 AGI fully eliminates the credit."
+us,scenario_088,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model checked only whether $18,577 of earned income was below an estimated EITC cutoff and then assigned nearly the maximum credit without applying the phaseout schedule. It also omitted that the phaseout uses the greater of earned income or approximately $24,035 AGI, which produces a $0 EITC."
 us,scenario_088,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model treated stated disability as sufficient for the aged, blind, or disabled pathway without applying its financial and categorical requirements. At roughly $23,375 of income and with no SSI receipt, the head qualifies through no Medicaid category."
 us,scenario_088,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model calculated SSI-style countable income of $13,561.50, recognized that it exceeded the approximately $11,604 SSI-linked limit, and then discarded that result to award eligibility based on disability and blindness alone. Those statuses do not create automatic SSI-related Medicaid eligibility, and the head receives no SSI."
 us,scenario_088,head_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model asserted that $31,776 in bank assets and the head’s countable income were within typical SSI-related Medicaid limits without applying the actual financial test. Disability and blindness only open the ABD pathway for evaluation; these facts do not satisfy it, and no other Medicaid category applies."
@@ -5290,34 +5557,36 @@ us,scenario_088,snap,claude-sonnet-4.6,llm_error,asset_resource,False,"The model
 us,scenario_088,snap,claude-sonnet-5,llm_error,asset_resource,False,"The model calculated a benefit from income and shelter deductions while omitting the $31,776 bank account from SNAP eligibility. That countable resource balance exceeds the applicable limit for a household with a disabled member, ending eligibility before its estimated $100 monthly benefit calculation; it also incorrectly applied a shelter-deduction cap despite the disabled member."
 us,scenario_088,snap,glm-5.2,llm_error,asset_resource,False,"The model treated high mortgage interest and property taxes as reducing net income to zero and then awarded the maximum allotment, without testing the $31,776 bank balance against the SNAP resource limit. The excess countable resources make the household ineligible regardless of the shelter deduction."
 us,scenario_088,snap,gpt-5.6-luna,llm_error,asset_resource,False,"The model moved directly from the disabled-member net-income pathway and large shelter deduction to the maximum one-person allotment, omitting the SNAP asset test. The head’s $31,776 in bank-account assets exceeds the applicable countable-resource limit, so no allotment is payable."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $3,467.43 qualified-overtime deduction, leaving taxable income at about $64,432 instead of $60,964.27. It then submitted $8,225 despite its own regular-tax calculation of roughly $7,236, so its final value also contradicts its stated computation."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented $57,893 of wages for the head even though no head wage or hourly rate was listed and unlisted income was explicitly zero. It also used the wrong standard deduction and unsupported premium adjustments instead of the QBI and overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used approximate values for the standard and QBI deductions and omitted the $3,467.43 overtime deduction. Its submitted $9,089 also fails to follow its own stated regular-income-tax result of about $7,377."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model added approximately $4,286 of self-employment tax to regular federal income tax even though this output excludes self-employment tax. It also omitted both the $9,568.44 QBI deduction and the $3,467.43 overtime deduction from taxable income."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's derivation reached roughly $7,150 of regular income tax but then submitted $12,850 based on an unsupported higher-effective-rate adjustment. It also omitted the $3,467.43 overtime deduction and used approximate deduction parameters."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the head's $5,403 desired traditional 401(k) contribution despite recognizing that the head had no wages from which to defer it. This understated AGI by $5,403, while its use of a $30,800 rather than $32,200 standard deduction only partially offset that error; it also omitted the overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated the $19,650 partnership/S-corp income as self-employment earnings, inflating self-employment tax and its associated deduction, and it failed to apply the QBI and overtime deductions. It then rounded its computed $8,054 upward to $8,300 without a supporting tax step."
-us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced wages to $57,735 instead of the trace's $58,934.98, used a $29,200 standard deduction, calculated QBI as 20% of unreduced business income, and omitted the overtime deduction. Those errors produced $65,802.90 of taxable income rather than $60,964.27."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and substituted a $16,000 standard deduction, personal exemptions, and a 15% bracket. It also double-deducted the listed employer-sponsored insurance premiums from wages and omitted the QBI and overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies a materially overstated taxable-income base: the correct deductions reduce $106,200.13 of AGI to $60,964.27. The model's abbreviated calculation omitted the specific $9,568.44 QBI deduction, $3,467.43 overtime deduction, and applicable 2026 deduction parameters."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied obsolete pre-TCJA rules by using a small standard deduction, personal exemptions, and pre-TCJA brackets. It also double-deducted $7,789 of employer-sponsored insurance premiums from wages and omitted the QBI and overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model wrongly applied a $15,300 standard deduction, personal exemptions, and a 15% bracket instead of the operative 2026 married-filing-jointly parameters. It also subtracted employer insurance premiums again from wages and omitted the QBI and overtime deductions."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model double-deducted the spouse's $7,789 employer-sponsored insurance premiums from wages, reducing AGI to $98,410.90 instead of $106,200.13. It also omitted the $3,467.43 overtime deduction, yielding taxable income below the correct $60,964.27."
-us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model used an AGI of about $98,663 rather than $106,200.13, implying an unsupported extra reduction of wage income such as the listed employer insurance premiums. The correct standard, QBI, and overtime deductions then yield $60,964.27 of taxable income and $6,819.71 of tax."
-us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $30,600 standard deduction rather than $32,200 and omitted the $3,467.43 overtime deduction. Consequently, it taxed $66,031.53 instead of $60,964.27."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,missing_output,False,"The model disregarded the explicit married household, annual income, deduction, and zero-default facts and set the liability to zero. Those supplied facts determine $60,964.27 of taxable income and a positive $6,819.71 federal income tax liability."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's $6,828 estimate is consistent with rounding the taxable-income inputs or 2026 bracket parameters instead of using the exact traced amounts. Exact taxable income of $60,964.27 under the applicable brackets produces $6,819.71."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model reported taxable income of about $62,996 rather than $60,964.27. Its reference to pretax health deductions reflects an unsupported treatment of the listed premiums, and its combined deduction arithmetic did not reproduce the exact $32,200 standard deduction, $9,568.44 QBI deduction, and $3,467.43 overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model named the standard and QBI deductions but omitted the qualified-overtime deduction from its stated computation. Its $7,530 answer corresponds to taxing a base materially above the correct $60,964.27."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used a rough combined-income figure near $120,000 rather than deriving $106,200.13 of AGI and $60,964.27 of taxable income. It failed to account specifically for the pretax 401(k), half-SE-tax and IRA adjustments, QBI deduction, and overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly imposed a TCJA sunset, replacing the $32,200 standard deduction and current brackets with personal exemptions, a $16,637 deduction, and a 15% bracket. It also omitted the $9,568.44 QBI deduction and $3,467.43 overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model added the $3,467 overtime premium on top of gross wages even though gross wages already include overtime, and it deducted the head's unfunded $5,403 401(k) contribution. It also omitted the QBI and overtime-income deductions and invented a $600 self-employed health-insurance deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model calculated QBI as 20% of the unreduced $49,985 business income instead of 20% of $47,842.18 after the allocable half-SE-tax deduction. More importantly, it omitted the separate $3,467.43 overtime deduction, so it taxed about $64,003 rather than $60,964.27."
-us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the $3,467 overtime premium by adding it to gross wages even though the prompt says gross wages include overtime, then deducted both spouses' desired 401(k) contributions despite the head having no wages. It also failed to apply both the $9,568.44 QBI deduction and the $3,467.43 overtime deduction and used incorrect standard-deduction and bracket parameters."
-us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model reproduced the core deductions and reached taxable income within 25 cents of the traced $60,964.27, but applied inaccurate 2026 bracket thresholds or rounding. Exact application of the brackets to $60,964.27 yields $6,819.71, not $6,827.74."
-us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated partnership/S-corp income as self-employment income when calculating the half-SE-tax deduction and deducted the head's $5,403 desired 401(k) contribution despite no head wages. It also calculated QBI from unreduced business income, used an undersized standard deduction, and omitted the overtime deduction."
-us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the head's $5,403 desired 401(k) contribution despite the absence of head wages and omitted both the $9,568.44 QBI deduction and $3,467.43 overtime deduction. Its tax formula also used inconsistent bracket constants, including a $2,783.50 base at a $23,200 threshold."
-us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model invented $132,619 of wages for the head by assigning the spouse's $27 hourly rate and overtime facts to the head, even though the head had no listed wage or hourly rate. It also invented children and child tax credits despite no children being listed and treated the half-SE-tax deduction as $14,180 rather than $2,143.10."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model derived $64,432 of taxable income because it omitted the $3,467.43 qualified-overtime deduction. It then submitted $8,225 despite its own bracket calculation yielding $7,236, so its final value also contradicts its stated computation."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model invented $57,893 of wages for the head even though no head wage or hourly rate was listed; the $27 hourly rate belongs to the spouse, whose annual wages were already supplied. It also omitted the QBI and qualified-overtime deductions and used the wrong standard deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used approximate, understated standard and QBI deductions and omitted the $3,467.43 qualified-overtime deduction. Its submitted $9,089 also contradicts its own stated regular-income-tax calculation of about $7,377."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,payroll_tax_base,False,"The model omitted both the $9,568.44 QBI deduction and the $3,467.43 qualified-overtime deduction from taxable income. It then added $4,286 of self-employment tax to the requested federal income-tax output even though self-employment tax is reported separately."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's own deductions and bracket calculation produced regular income tax near $7,150, but it replaced that result with $12,850 based on an unsupported higher-effective-rate adjustment. The requested output excludes self-employment tax and contains no such adjustment."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted the head's $5,403 desired traditional 401(k) contribution despite the head having no wages, reducing AGI to $100,797 instead of $106,200.13. It also omitted the $3,467.43 overtime deduction and used estimated rather than actual 2026 deduction and bracket parameters."
+us,scenario_089,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model treated the $19,650 partnership/S-corp income as self-employment earnings, inflating self-employment tax and its above-the-line deduction. It also omitted the QBI and qualified-overtime deductions and used an understated standard deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced wages to $57,735 without support, used a $29,200 standard deduction, calculated QBI as 20% of unreduced business income, and omitted the $3,467.43 overtime deduction. The trace instead yields AGI of $106,200.13 and taxable income of $60,964.27."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and therefore used a $16,000 standard deduction, personal exemptions, and obsolete 10%/15% brackets. It also deducted the listed employer-sponsored insurance premium from wages even though the engine's wage computation does not do so."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer does not account for the traced $9,568.44 QBI deduction and $3,467.43 qualified-overtime deduction that reduce taxable income to $60,964.27. Its $11,090 result is inconsistent with applying the 2026 joint brackets to that taxable income."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly applied post-sunset pre-TCJA rules, including personal exemptions, a reduced standard deduction, and obsolete brackets. It also improperly subtracted $7,789 of employer-sponsored insurance premiums from wages and omitted the QBI and overtime deductions."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used obsolete personal exemptions, an understated standard deduction, and pre-TCJA-style 10%/15% brackets. It also improperly reduced wages by the listed employer insurance premium and omitted the QBI and qualified-overtime deductions."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,789 employer-sponsored insurance premium from wages, lowering AGI to $98,410.90. It also omitted the $3,467.43 qualified-overtime deduction; the correct deductions from $106,200.13 of AGI yield $60,964.27 of taxable income."
+us,scenario_089,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's stated AGI of about $98,663 reflects an unsupported reduction from the traced $106,200.13. It failed to preserve the engine's wage treatment and the exact standard, QBI, and overtime deductions that produce $60,964.27 of taxable income."
+us,scenario_089,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly derived AGI and QBI but used a $30,600 standard deduction instead of $32,200 and omitted the $3,467.43 qualified-overtime deduction. Those errors overstated taxable income by $5,067.43."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model disregarded the prompt's explicit married household, annual income, zero-default rule, and filing assumption and therefore returned zero. The supplied facts fully determine $106,200.13 of AGI, $60,964.27 of taxable income, and positive tax of $6,819.71."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model gave no usable computation and returned $6,828 instead of applying the exact 2026 joint brackets to $60,964.27 of taxable income. That bracket calculation yields $6,819.71, so the answer reflects approximate parameters or arithmetic."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model stated taxable income of about $62,996 rather than the traced $60,964.27. It failed to apply the exact $32,200 standard deduction, $9,568.44 QBI deduction, and $3,467.43 overtime deduction to AGI of $106,200.13."
+us,scenario_089,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model mentioned the standard and QBI deductions but omitted the qualified-overtime deduction from its stated computation. Applying all three deductions produces $60,964.27 of taxable income and $6,819.71 of tax, not $7,530."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used a rough combined-income shortcut and did not apply the traced above-the-line adjustments, $32,200 standard deduction, $9,568.44 QBI deduction, and $3,467.43 overtime deduction. Those steps reduce taxable income to $60,964.27 rather than an amount supporting $12,450 of tax."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset and applied personal exemptions, a $16,637 standard deduction, and obsolete 10%/15% brackets. Current 2026 rules instead provide the $32,200 standard deduction and 10%/12% brackets applicable to this taxable income."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly used post-sunset personal exemptions, a $16,600 standard deduction, and obsolete 10%/15% brackets. It also omitted the QBI and qualified-overtime deductions that are part of the 2026 taxable-income calculation."
+us,scenario_089,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model double-counted the $3,467 overtime premium by adding it to wages even though gross wages already include overtime, deducted the head's unsupported 401(k) contribution, and invented a self-employed health-insurance deduction. It also omitted both the QBI deduction and the statutory overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model calculated QBI as 20% of the unreduced $49,985 business income instead of 20% of $47,842.18 after the allocable half-SE-tax deduction. More importantly, it omitted the $3,467.43 qualified-overtime deduction, leaving taxable income too high."
+us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the $3,467 overtime premium even though the prompt says gross wages include overtime, and it deducted the head's $5,403 desired 401(k) contribution despite no head wages. It also omitted the $9,568.44 QBI deduction and the $3,467.43 qualified-overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,other,False,"The model closely reproduced the derivation but rounded the overtime deduction to $3,467, producing taxable income $0.25 above the traced amount. It then made an $8 arithmetic or transcription error in the bracket calculation: tax on approximately $60,964.5 under the stated 2026 brackets is about $6,819.74, not $6,827.74."
+us,scenario_089,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,payroll_tax_base,False,"The model computed the half-self-employment-tax deduction as about $3,531 by treating partnership/S-corp income as subject to self-employment tax. It also deducted the head's unsupported 401(k), used an understated standard deduction, overstated QBI, and omitted the qualified-overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model deducted both spouses' desired 401(k) contributions even though the head had no wages, lowering AGI to $100,797. It also calculated QBI from unreduced business income and omitted the $3,467.43 qualified-overtime deduction."
+us,scenario_089,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model deducted the head's $5,403 desired 401(k) contribution despite the absence of head wages and omitted both the $9,568.44 QBI deduction and the $3,467.43 overtime deduction. It also used an incorrect first-bracket tax amount inconsistent with its stated $23,200 threshold."
+us,scenario_089,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model invented $132,619 of head wages by applying the spouse's $27 hourly rate and work schedule to the head, even though no head wage was listed. It also invented four child credits despite no children in the household and used an erroneous $14,180 half-SE-tax deduction."
 us,scenario_089,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model awarded a $2,242.80 Additional Child Tax Credit despite the household having no qualifying children. It also fabricated $120,000 of head wages from hours worked and mislabeled $2,400 of supposed excess Additional Medicare withholding as a refundable credit, even though no head wage or hourly rate was listed and tax withholding refunds are not included in federal_refundable_credits."
 us,scenario_089,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model invented wage income for the head from hours worked despite the instruction that unlisted numeric inputs are zero, then invented North Carolina employee payroll taxes. It also initially characterized self-employment and partnership income as W-2 wages, although those amounts do not enter employee payroll tax."
 us,scenario_089,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly identified the $65,881 payroll-tax base and the applicable 6.2% and 1.45% rates, but its arithmetic was internally inconsistent and it ultimately submitted $5,085.44 without any supporting computation. The traced components total $5,039.91."
@@ -5347,35 +5616,37 @@ us,scenario_089,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_out
 us,scenario_089,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model incorrectly treated all $19,650 of the combined partnership-or-S-corporation income field as additional self-employment earnings. PolicyEngine applies self-employment tax to the separately specified $30,335 self-employment-income input, producing $4,286.24."
 us,scenario_089,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model recognized that S-corporation income is not subject to self-employment tax but nevertheless classified the entire ambiguous combined partnership-or-S-corporation amount as partnership income and added it to the base. The engine uses the explicit $30,335 self-employment-income input and derives $4,286.24."
 us,scenario_089,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly combined the $30,335 self-employment income with the $19,650 partnership-or-S-corporation amount and then invented a $668.78 offset for employee Social Security tax. W-2 Social Security withholding does not create a dollar-for-dollar offset against self-employment tax; wages only reduce the remaining Social Security wage-base capacity, which is not exhausted here."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's stated derivation reaches roughly $3,230 but it submitted $4,022 instead. It violated its own final computation rather than applying 3.99% to the traced $80,700.13 taxable income."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model never completed the NC taxable-income calculation and submitted a liability inconsistent with the applicable base. Federal AGI is $106,200.13, so subtracting the $25,500 joint standard deduction and applying 3.99% yields $3,219.94."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"Its own successive calculations produced about $3,004 to $3,200, but it submitted $4,710 without a supporting computation. It also improperly deducted the head's unfunded 401(k) contribution from income despite the head having no wages."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly reconstructed approximately $80,700 of taxable income and approximately $3,220 of tax, then abandoned that result and invented a $132,500 taxable base. The supplied income facts do not produce that larger base."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model changed taxable income from about $81,000 to $87,100 without identifying any NC addition. The correct base is $106,200.13 minus $25,500, or $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a 4.25% rate and a fabricated $30,000 joint standard deduction. For 2026 this calculation uses the 3.99% rate and $25,500 deduction, producing $3,219.94 on $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly multiplied the combined self-employment and partnership income by 92.35% and computed the half-SE-tax deduction on both amounts. Partnership or S-corporation income is not automatically reduced that way, and the traced federal AGI is $106,200.13."
-us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $1,200.23, using $104,999.90 instead of $106,200.13. That carried directly into an understated NC taxable-income base."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $7,789.13, exactly consistent with deducting one listed employer-sponsored insurance premium from the income base. That premium is not an additional deduction from the traced AGI."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,410 implies a taxable base near $110,526 at 3.99%, but the traced base after the joint standard deduction is $80,700.13. The model omitted the specific income and deduction computation needed to support its inflated base."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used federal AGI of $98,411 instead of $106,200.13. The $7,789.13 understatement reflects treating a listed employer-sponsored insurance premium as an extra AGI deduction."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model both understated AGI to $98,411 and used an incorrect $29,250 NC joint standard deduction. The applicable figures are $106,200.13 and $25,500, respectively."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated NC AGI by $7,789.23, consistent with subtracting one employer-sponsored insurance premium as an additional deduction. NC taxable income is $80,700.13, not $72,910.90."
-us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"At 3.99%, the answer implies taxable income of about $73,163 rather than $80,700.13. The model therefore understated the AGI-derived NC tax base before applying the flat rate."
-us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model reduced self-employment income to 92.35% and then also subtracted half of self-employment tax, double-reducing that income component. The traced AGI is $106,200.13, not $103,879.36."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The household has $80,700.13 of NC taxable income after the joint standard deduction, so the standard deduction does not eliminate liability. Applying 3.99% produces $3,219.94 rather than zero."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The submitted amount implies taxable income of only about $76,566 at 3.99%. The model over-deducted roughly $4,134 instead of using the traced $106,200.13 AGI and $25,500 standard deduction."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted the listed employer-sponsored insurance premium from wages in addition to the adjustments already reflected in the traced AGI. That reduced AGI to $98,411 instead of $106,200.13."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $69,696 instead of $80,700.13. It therefore introduced $11,004.13 of unsupported additional deductions before applying the correct rate."
-us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The rounded estimate did not preserve the traced AGI and exact taxable-income computation. Applying 3.99% to $106,200.13 minus $25,500 yields $3,219.94, not $3,230."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The answer implies taxable income of about $78,195 at 3.99%, understating the correct $80,700.13 base. The model omitted the AGI and deduction calculation that would have prevented that understatement."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly identified AGI near $106,200 and the $25,500 deduction but applied 4.25%. The applicable 2026 NC rate is 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $103,664 rather than $106,200.13 and rounded the statutory rate to 4%. The exact computation uses $80,700.13 of taxable income at 3.99%."
-us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $26,700 joint standard deduction and additional unspecified NC adjustments. The applicable deduction is $25,500, leaving $80,700.13 taxable."
-us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
-us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated federal AGI by $3,467.23, effectively removing the overtime-premium amount from the annual wage base. Gross wages already include overtime, and the traced AGI is $106,200.13."
-us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a 4.5% NC rate instead of the applicable 3.99% rate and also understated AGI. The correct taxable base is $80,700.13."
-us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model used a 4.5% rate instead of 3.99%, added the separately listed overtime premium on top of annual gross wages even though gross wages include overtime, and deducted the head's unfunded 401(k). These errors produced the wrong AGI and tax rate."
-us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented $248,485 of AGI, a $32,200 standard deduction, and an out-of-state tax credit unsupported by the household facts. The only applicable state calculation starts from $106,200.13 of AGI, subtracts $25,500, and taxes the result at 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's revised inputs were close to the traced calculation, but its submitted $4,022 contradicts its own $3,230 arithmetic. It failed to carry its final taxable-income-and-rate computation into the output."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model never formed the required $106,200.13 NC AGI or subtracted the $25,500 joint standard deduction. Its $4,532 output therefore applies the state tax to an overstated or insufficiently reduced income base."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model calculated figures of roughly $3,004 to $3,200, then submitted $4,710 without any supporting computation. It failed to transfer its own tax calculation to the output."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly reconstructed approximately $80,700 of taxable income and approximately $3,220 of tax, then abandoned that derivation for an invented $132,500 base. The submitted $5,290 is unsupported by the listed income and contradicts its own calculation."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model switched from taxable income near $81,000 to an unexplained $87,100 base. The applicable base is $106,200.13 minus $25,500, or $80,700.13."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $30,000 standard deduction and a 4.25% rate. The applicable joint deduction is $25,500 and the 2026 NC rate is 3.99%, producing tax on $80,700.13."
+us,scenario_089,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly multiplied the combined self-employment and partnership income by 92.35% and also entertained subtracting the head's unsupported 401(k) contribution, depressing its AGI calculation. It then submitted $3,800 despite its own arithmetic yielding about $2,979, so the final output also contradicts its derivation."
+us,scenario_089,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated NC AGI as $104,999.90 instead of $106,200.13. With the correct $25,500 deduction, the 3.99% rate applies to $80,700.13 rather than $79,499.90."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model understated federal/NC AGI as $98,411. It should start from $106,200.13, leaving $80,700.13 after the $25,500 joint standard deduction."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,410 implies an NC taxable-income base of about $110,526 at the 3.99% rate. The correct base after the joint standard deduction is $80,700.13."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated NC AGI as $98,411 instead of $106,200.13. Subtracting $25,500 from the correct AGI yields $80,700.13, not $72,911."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model both understated AGI as $98,411 and used an incorrect $29,250 joint standard deduction. The calculation requires $106,200.13 of AGI less the $25,500 deduction."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated NC AGI as $98,410.90. The traced AGI is $106,200.13, so taxable income is $80,700.13 after the $25,500 deduction."
+us,scenario_089,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $2,919.20 answer implies taxable income of only about $73,163 at 3.99%. The correct taxable income is $80,700.13 after subtracting the $25,500 joint deduction from $106,200.13."
+us,scenario_089,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model reduced self-employment income to 92.35% before also subtracting half of self-employment tax, thereby double-reducing the AGI income component. NC AGI is $106,200.13, not $103,879.36."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the household's substantial income as eliminated by deductions and returned zero. NC taxable income remains $80,700.13 after the $25,500 joint standard deduction, producing positive tax."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The $3,055 answer implies taxable income of about $76,566 at 3.99%, understating the base by roughly $4,134. The traced taxable income is $80,700.13."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted employer-sponsored insurance premiums from the stated gross wages and thereby reduced AGI to $98,411. The trace uses NC AGI of $106,200.13 before the $25,500 state standard deduction."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $69,696 instead of $80,700.13. It therefore incorporated approximately $11,004 of unsupported additional income reductions."
+us,scenario_089,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,other,False,"The stated method is essentially the correct one, but the submitted rounded estimate does not preserve the engine's cents-level computation. Applying 3.99% to $80,700.13 yields $3,219.94, not $3,230."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $3,120 answer implies taxable income of about $78,195 at 3.99%, below the traced $80,700.13 base. The model omitted or over-deducted roughly $2,505 of taxable income."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model correctly used approximately $106,200 of AGI and the $25,500 deduction but applied a 4.25% rate. The applicable 2026 NC rate is 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly reached approximately $80,700 of taxable income but applied 4.25%. Applying the 2026 rate of 3.99% yields $3,219.94."
+us,scenario_089,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $103,664 rather than $106,200.13 and consequently used only $78,164 of taxable income. The applicable taxable base is $80,700.13."
+us,scenario_089,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used an unsupported $26,700 joint standard deduction instead of $25,500. Starting from $106,200.13, the correct deduction leaves $80,700.13 taxable."
+us,scenario_089,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_089,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated NC AGI as $102,732.90 instead of $106,200.13. The correct taxable income after the $25,500 deduction is $80,700.13, not $77,232.90."
+us,scenario_089,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used a 4.5% tax rate instead of the 2026 NC rate of 3.99%, while also understating AGI. The correct computation applies 3.99% to $80,700.13."
+us,scenario_089,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model improperly deducted both spouses' desired traditional 401(k) contributions and both IRA contributions, reducing AGI to $100,797. The traced NC AGI is $106,200.13, leaving $80,700.13 taxable after the state deduction."
+us,scenario_089,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,thresholds_rates,False,"The model double-counted the separately listed FLSA overtime premium as additional wages, miscomputed the half-SE-tax deduction, and deducted the head's unsupported 401(k) contribution. It then applied an incorrect 4.5% rate rather than 3.99%."
+us,scenario_089,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented $248,485 of AGI, a $32,200 standard deduction, and an out-of-state tax credit unsupported by the household facts. The calculation instead uses $106,200.13 of AGI, a $25,500 deduction, and no such credit."
 us,scenario_089,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_090,federal_refundable_credits,claude-opus-4.8,llm_error,age_disability,False,"The model treated the 22-year-old as eligible for the childless EITC, missing its minimum age of 25. Its subsequent phase-in and phaseout estimate is irrelevant because age disqualifies the filer before the credit schedule is applied."
 us,scenario_090,federal_refundable_credits,claude-sonnet-5,llm_error,age_disability,False,"The model applied the childless EITC schedule to a 22-year-old even though the minimum qualifying age is 25. It also submitted $721 after calculating a $535.50 phase-in amount, but the controlling error is that no EITC schedule applies to this filer."
@@ -5402,49 +5673,53 @@ us,scenario_090,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The mod
 us,scenario_090,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used the generic federal gross-income screen instead of the applicable Kansas eligibility threshold and therefore treated the household as entitled to a positive allotment. Its submitted $2,650 is also unsupported by its own calculation of about $99 per month, or $1,188 annually."
 us,scenario_090,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,The model inferred eligibility from the household's seemingly low cash income without applying the Kansas SNAP eligibility threshold. PolicyEngine's eligibility calculation yields $0 before any positive allotment can be computed.
 us,scenario_090,snap,grok-build-0.1,llm_error,thresholds_rates,False,"The model assumed the household passed SNAP eligibility, then mechanically subtracted 30% of deducted net income from the maximum allotment using FY2025 parameters. It missed the prior Kansas threshold test that makes the household ineligible and prevents any allotment calculation."
-us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $3,500 standard deduction and $2,250 personal exemption, leaving $1,270 of artificial taxable income. The applicable 2026 Kansas deduction threshold fully offsets the $7,020 of AGI, so there is no amount to tax at 5.2% or any other rate."
-us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only an obsolete $3,500 standard deduction and omitted the remaining Kansas deduction or exemption treatment that reduces taxable income to zero. Its $109.12 is simply 3.1% of the resulting artificial $3,520 tax base."
-us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used obsolete Kansas deductions of $3,500 and $2,320 and therefore manufactured $1,200 of taxable income. Under the applicable 2026 deduction threshold, the full $7,020 of AGI is sheltered, so its subsequent 5.2% calculation has no tax base."
-us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly limited the Kansas standard deduction and personal exemption to $3,600 and $2,320, leaving $1,100 taxable. The applicable 2026 deduction threshold eliminates taxable income at $7,020 of AGI, making the 5.2% rate calculation inapplicable."
-us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used obsolete deduction inputs of $3,605 and $2,250, producing a nonexistent $1,165 Kansas tax base. The applicable 2026 deduction threshold reduces taxable income to zero before any marginal rate is applied."
-us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model relied on obsolete $3,500 and $2,250 deduction amounts and consequently treated $1,270 as taxable income. Kansas’s applicable 2026 deduction threshold shelters the entire $7,020 of AGI, so the 3.1% calculation should never occur."
-us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted only an obsolete $3,500 standard deduction, leaving an artificial $3,520 tax base and taxing it at 3.1%. The applicable 2026 Kansas deduction threshold reduces taxable income to zero."
-us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model’s asserted $1,120 of taxable income comes from understated, unspecified Kansas deductions. Applying the 2026 deduction threshold to $7,020 of AGI leaves zero taxable income, so its 5.2% multiplication is inapplicable."
-us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used obsolete deductions of $3,605 and $2,250, leaving $1,165 of artificial taxable income. The applicable 2026 Kansas deduction threshold fully offsets the household’s $7,020 AGI, leaving nothing to tax at 5.2%."
-us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model subtracted only an obsolete $3,500 standard deduction, creating a $3,520 tax base, and then compounded the error by applying a 5.7% rate. The applicable 2026 Kansas deduction threshold reduces taxable income to zero before rates are considered."
-us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model approximated obsolete deductions at $3,600 and $2,300, leaving about $1,100 of taxable income that does not exist under the 2026 rules. The applicable Kansas deduction threshold shelters all $7,020 of AGI, so applying an approximate 5.5% rate was erroneous."
+us,scenario_090,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an obsolete $3,500 standard deduction and $2,250 personal exemption, then applied a speculative 5.2% rate to the resulting $1,270. Under the applicable 2026 Kansas standard-deduction threshold, the full $7,020 is sheltered and no positive taxable base remains."
+us,scenario_090,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only a $3,500 standard deduction, omitted the remaining applicable Kansas income shelter, and taxed $3,520 at 3.1%. The 2026 Kansas deduction threshold reduces taxable income to zero before any rate is applied."
+us,scenario_090,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used outdated deduction and exemption amounts, asserted that a $7,000 low-income cutoff controlled, and applied a 5.2% flat rate to $1,200. The applicable 2026 Kansas standard-deduction threshold shelters the household’s entire $7,020 of income."
+us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied estimated Kansas deductions of $3,600 and $2,320 and then taxed a manufactured $1,100 base at 5.2%. The applicable 2026 standard-deduction threshold eliminates the taxable base entirely."
+us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used obsolete deduction parameters and the former 3.1% bottom-bracket rate, leaving $1,165 taxable. The applicable 2026 Kansas standard-deduction threshold reduces taxable income to zero, so no bracket rate applies."
+us,scenario_090,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a $3,500 standard deduction, a $2,250 exemption, and the obsolete 3.1% bottom rate to tax $1,270. The applicable 2026 Kansas standard-deduction threshold shelters all $7,020 before the rate calculation."
+us,scenario_090,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted only an obsolete $3,500 standard deduction and treated the remaining $3,520 as taxable. The applicable 2026 Kansas deduction threshold leaves zero taxable income, making its 3.1% rate calculation inapplicable."
+us,scenario_090,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model created a $1,120 taxable base from unspecified estimated deduction and exemption amounts and applied 5.2%. The applicable 2026 Kansas standard-deduction threshold shelters the entire $7,020, leaving no amount to tax."
+us,scenario_090,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used obsolete Kansas deduction and exemption figures to leave $1,165 taxable, then applied 5.2%. The applicable 2026 standard-deduction threshold instead reduces taxable income to zero."
+us,scenario_090,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model relied on approximate, obsolete Kansas deduction parameters to produce a $1,165 taxable base and tax it at 5.2%. The applicable 2026 standard-deduction threshold fully shelters the household’s income."
+us,scenario_090,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model subtracted only an obsolete $3,500 deduction and then applied an inapplicable 5.7% flat rate to $3,520. The applicable 2026 Kansas standard-deduction threshold leaves zero taxable income before rates."
+us,scenario_090,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model substituted approximate deduction, exemption, and rate figures for the applicable 2026 Kansas parameters, producing an artificial taxable base near $1,100. The 2026 standard-deduction threshold shelters all $7,020, so the approximate 5.5% calculation never arises."
+us,scenario_090,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $3,500 standard deduction, taxed the remaining $3,520, and treated the personal exemption as a $110 credit. The applicable 2026 Kansas standard-deduction threshold reduces taxable income to zero before either the 5.2% rate or a personal-exemption credit calculation."
 us,scenario_090,state_refundable_credits,glm-5.2,llm_error,categorical_eligibility,False,"The model applied the federal EITC phase-in rate without checking the childless claimant age rule. At age 22 with no qualifying child, the head receives no federal EITC, so applying Kansas’s percentage to a fabricated $535.50 federal credit incorrectly produced $96.39."
 us,scenario_090,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model treated low earned income as sufficient for the federal childless EITC and then applied a Kansas percentage. The 22-year-old head is below the minimum age for a claimant without a qualifying child, leaving no federal EITC on which to calculate a Kansas credit."
 us,scenario_090,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,"The model calculated a childless federal EITC of $536 but omitted the federal minimum-age condition. Because the head is 22 and has no qualifying child, the federal EITC is zero and the derivative Kansas credit is also zero."
 us,scenario_090,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model treated the Kansas Food Sales Tax Credit’s income ceiling as its sole eligibility test. A claimant must also be age 55 or older, disabled or blind, or have a dependent child under 18; this 22-year-old single filer meets none of those categories, so the $125 credit is unavailable."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model included the $1,170 non-Schedule-D capital gain in gross income even though the engine's traced gross income is $35,055.84, and it omitted the $627.91 overtime deduction. Its stated recomputations also never support its submitted $1,276 value."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model invented a full $2,500 American Opportunity Credit despite the prompt setting unlisted qualified education expenses to zero. It also used the wrong standard deduction and taxed qualified dividends at 15% instead of applying the preferential stacking calculation."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the $1,170 capital gain in AGI, omitted the $627.91 overtime deduction, and failed to subtract the $120 saver's credit. Those errors replaced the traced $1,470 tentative tax and $1,350 post-credit result with $1,656."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model treated stated gross wages as already net of the traditional 401(k), included the $1,170 capital gain in AGI, and omitted the overtime deduction. It then arbitrarily applied $331 of an AOTC unsupported by any listed education expense instead of the actual $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model's own preferential-rate calculation produced about $1,684 before credits, but it discarded that result and submitted $2,109 by taxing the full taxable-income base inconsistently. It also omitted the overtime deduction and $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $15,000 standard deduction rather than $16,100, included the $1,170 capital gain in income, and omitted the $627.91 overtime deduction. It also failed to apply the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model discussed an AOTC despite zero listed qualified education expenses and then submitted a number unrelated to its own claimed $1,500 nonrefundable AOTC reduction. It also used the wrong standard deduction, included the $1,170 gain, and omitted the actual $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $1,800 health-insurance premium from wages and used a $15,000 standard deduction, while also including the $1,170 capital gain and omitting the overtime deduction. The near-match is accidental; the traced taxable-income calculation yields $1,470 before the $120 saver's credit, not $1,470.90 before a $119.90 credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model invented an American Opportunity Credit with no listed qualified education expense and used it to erase the liability. It also deducted the $6,589 employer-sponsored premium from stated gross wages and used an obsolete personal exemption."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted explanation implies taxable income of about $25,500, whereas the traced deductions and qualified-dividend treatment produce the tax base leading to $1,470 before credits. It failed to account for the $16,100 standard deduction, $627.91 overtime deduction, and $120 saver's credit correctly."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $6,589 employer-sponsored insurance premium from gross wages and applied a personal exemption. Those unsupported deductions understated AGI and ordinary taxable income, producing $896 instead of $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model deducted the $6,589 employer-sponsored premium from stated gross wages and applied an $8,300 standard deduction plus a $5,300 personal exemption instead of the traced $16,100 standard deduction. It also omitted the overtime deduction and $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $6,589 employer-sponsored insurance premium from gross wages and relied on a personal exemption. It did not follow the traced $16,100 standard deduction, $627.91 overtime deduction, and $120 saver's-credit calculation."
-us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model submitted $1,086 while its explanation ended with value = 1756, violating the required agreement between the numeric value and explanation. Neither number follows the traced $1,470 tentative tax less the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used a $15,400 standard deduction, included the $1,170 capital gain in AGI, and omitted the $627.91 overtime deduction. Its separate saver's-credit AGI calculation also subtracted the retirement exclusions a second time."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model invented an American Opportunity Credit despite zero listed qualified education expenses and used it to eliminate the tax. The standard deduction does not fully offset the traced income; the remaining computation produces $1,470 before the $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model treated an unsupported American Opportunity Credit as available and concluded that nonrefundable credits erased the liability. With no listed qualified education expense, the applicable nonrefundable reduction is the $120 saver's credit, leaving $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model reached the correct $1,470 tax before credits but failed to apply the $120 nonrefundable saver's credit. It also described AGI as $36,183 rather than the traced $35,012.56 because it included the $1,170 capital gain."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model invented a nonrefundable American Opportunity Credit with no listed qualified education expense and used it to reduce tax to zero. The actual applicable nonrefundable credit is the $120 saver's credit, leaving $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"Although the model recognized the standard deduction, overtime deduction, zero-rate treatment for qualified dividends, and absence of AOTC expenses, it computed the resulting bracket tax incorrectly. The traced bracket calculation is $1,470, followed by the omitted $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model invented an American Opportunity Credit and unspecified other nonrefundable credits to erase the liability. No qualified education expense was listed, and the actual $120 saver's credit reduces $1,470 only to $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied restored post-TCJA 10% and 15% brackets and a personal exemption rather than the 2026 parameters used in the trace. It also included the $1,170 capital gain in AGI and omitted the $627.91 overtime deduction."
-us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly deducted the $6,589 employer-sponsored premium from gross wages and used an $8,350 standard deduction plus a $5,330 personal exemption. It also omitted the $627.91 overtime deduction and $120 saver's credit."
-us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an obsolete smaller standard deduction, a personal exemption, and restored 15% brackets instead of the traced 2026 deduction and rates. It also estimated the saver's credit as $97 rather than the actual $120 and omitted the overtime deduction."
-us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
-us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model used the American Opportunity Credit to eliminate tax despite zero listed qualified education expenses. The refundable portion cannot reduce the requested before-refundable-credits output, and the actual nonrefundable saver's credit leaves $1,350."
-us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model double-counted the $628 overtime premium on top of annual wages even though wages already include it, then invented education expenses and an AOTC. The overtime premium instead generates a $627.91 deduction, and no AOTC is allowed because qualified education expenses are unlisted and therefore zero."
-us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model asserted that the single standard deduction reduced $40,571 of taxable income to zero, an arithmetic impossibility and an income figure unsupported by the facts. The traced deductions leave income subject to tax, producing $1,470 before the $120 saver's credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It incorrectly included the $1,170 non-Schedule-D gain in gross income and omitted the $627.91 qualified-overtime deduction. Its submitted $1,276 also does not follow from any of the successive calculations in its explanation."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,It invented a full American Opportunity Tax Credit despite the instruction that unlisted qualified tuition expense is zero. It also omitted the pre-tax 401(k) exclusion and overtime deduction and taxed qualified dividends at 15% even though they fall in the 0% band.
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It included the $1,170 non-Schedule-D gain in AGI, omitted the $627.91 overtime deduction, and failed to subtract the $120 Saver’s Credit. Those errors leave its ordinary taxable-income base above the engine’s base."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"It treated the stated $27,000 gross wages as already net of the traditional 401(k), included the $1,170 gain, and omitted the overtime deduction. It then fabricated a $331 AOTC amount even though no qualified education expense was listed."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"Its own preferential-rate computation produced tax near $1,684, but it submitted $2,109 without a valid calculation. It also included the $1,170 gain, omitted the overtime deduction, and did not apply the $120 Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"It used an incorrect $15,000 standard deduction and included the $1,170 gain in taxable income while omitting the qualified-overtime deduction. It also failed to subtract the $120 Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"It included the $1,170 gain, used the wrong standard deduction, and did not apply the overtime deduction or Saver’s Credit. It also invoked a nonexistent AOTC based solely on enrollment facts despite zero listed qualified education expense, and its final number does not follow from its stated arithmetic."
+us,scenario_091,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It improperly deducted the $1,800 other-health-insurance premium from AGI and used a $15,000 standard deduction instead of $16,100. It also included the $1,170 gain and omitted the $627.91 overtime deduction; the near match is cancellation among these errors."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"It wrongly deducted $6,589 of employer-sponsored insurance premiums from the stated gross wages and then invented an AOTC with no listed qualified education expense. It also used obsolete deduction assumptions instead of the $16,100 standard deduction and omitted the overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its claimed taxable income of about $25,500 omits the applicable $16,100 standard deduction and $627.91 overtime deduction computation. The correct ordinary-rate base produces $1,470 before the $120 Saver’s Credit, not $1,399."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"It wrongly subtracted $6,589 of employer-sponsored insurance premiums from gross wages and applied a pre-TCJA standard-deduction-and-exemption structure. The engine instead uses the $16,100 standard deduction plus the overtime deduction before applying the Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It improperly deducted $6,589 of employer-sponsored insurance premiums from wages and used an obsolete $8,300 standard deduction plus $5,300 personal exemption. It also omitted the qualified-overtime deduction and $120 Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"It wrongly deducted the $6,589 employer-sponsored insurance premium and applied an obsolete personal exemption. It omitted the engine’s $16,100 standard deduction, $627.91 overtime deduction, and $120 Saver’s Credit sequence."
+us,scenario_091,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"Its explanation states a final calculation of $1,756 but submits $1,086, so the submitted output contradicts its own derivation. That derivation also includes the $1,170 gain and omits the overtime deduction and Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"It used a $15,400 standard deduction rather than $16,100, included the $1,170 gain, and omitted the $627.91 overtime deduction. Those taxable-base errors overstate tentative tax despite correctly approximating the Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"It treated the AOTC as available without any listed qualified education expense and therefore erased the liability. The standard and overtime deductions leave positive taxable income, and only the $120 Saver’s Credit reduces the $1,470 tentative tax."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"It applied an American Opportunity Credit despite zero listed qualified education expense. The available nonrefundable credit is the $120 Saver’s Credit, which reduces tentative tax only to $1,350."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"It correctly approached the $1,470 tentative tax but stopped before subtracting the $120 nonrefundable Saver’s Credit. It also incorrectly included the $1,170 gain in AGI and in the preferential-income subtraction, obscuring the correct traced base."
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,It invented a nonrefundable American Opportunity Credit without listed qualified education expenses and used it to reduce tax to zero. The actual nonrefundable reduction is the $120 Saver’s Credit.
+us,scenario_091,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"Although it recognized the standard and overtime deductions and rejected the AOTC, it failed to subtract the $120 Saver’s Credit and overstated the ordinary-rate computation. The traced tentative tax is $1,470 before that credit and $1,350 afterward."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"It used an AOTC to eliminate the tax despite zero listed qualified education expense. The deductions do not eliminate taxable income, and the $120 Saver’s Credit is the only traced nonrefundable credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It incorrectly applied a post-TCJA-sunset personal exemption and restored 15% bracket instead of the engine’s 2026 $16,100 standard deduction and applicable 10%/12% brackets. It also included the $1,170 gain and omitted the overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"It used an obsolete $8,300 standard deduction, personal exemption, and 15% second bracket rather than the engine’s 2026 deduction and rate parameters. It also included the $1,170 gain and omitted the qualified-overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It improperly subtracted $6,589 of employer-sponsored insurance premiums from stated gross wages and used an obsolete standard deduction plus personal exemption. It also included the $1,170 gain, omitted the overtime deduction, and failed to apply the Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"It applied an obsolete standard deduction, personal exemption, and 15% bracket instead of the engine’s $16,100 standard deduction and 10%/12% brackets. It also included the $1,170 gain, omitted the overtime deduction, and understated the Saver’s Credit."
+us,scenario_091,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It returned no numeric output or explanation for the requested variable.
+us,scenario_091,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,It applied an American Opportunity Credit despite zero listed qualified education expense and incorrectly reduced the liability to zero. The traced nonrefundable credit is only the $120 Saver’s Credit.
+us,scenario_091,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"It assigned a 50% Saver’s Credit rate even though the applicable rate is 10%, producing $484.50 instead of $120. It also included the $1,170 gain and omitted the $627.91 overtime deduction."
+us,scenario_091,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"It double-counted the $628 overtime premium on top of annual gross wages even though wages already include it, then invented qualified education expenses and an AOTC. It also included the $1,170 gain and failed to apply the overtime-income deduction correctly."
+us,scenario_091,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It asserted that the single standard deduction reduces more than $35,000 of AGI to zero, which is arithmetically false. The $16,100 standard deduction and $627.91 overtime deduction leave positive taxable income, yielding $1,350 after the Saver’s Credit."
 us,scenario_091,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived a $0 refundable AOTC from $0 qualified education expenses, then discarded that derivation and inserted a $400 “minimal refundable AOTC assumption.” The AOTC has no minimum refundable amount, so $0 of expenses produces $0 of refundable credit."
 us,scenario_091,federal_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model treated enrollment, credential, 1098-T, and institution-EIN facts as sufficient to generate an AOTC despite the prompt setting unlisted qualified education expenses to $0. It also misstated the refundable AOTC calculation: the refundable portion is 40% of the expense-based AOTC, capped at $1,000, not $1,200 or 15% of education expenses."
 us,scenario_091,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model awarded the maximum $1,000 refundable AOTC solely from student-status and documentation conditions. Those conditions establish only potential eligibility; with no listed qualified education expenses, the AOTC base is $0 and its refundable portion is $0."
@@ -5469,36 +5744,38 @@ us,scenario_091,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The
 us,scenario_091,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax value or explanation. It therefore failed the required structured-output contract rather than performing a reviewable payroll-tax computation.
 us,scenario_091,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model added the $628 FLSA overtime premium to the $27,000 gross-wage amount even though the prompt states that gross annual wages already include overtime pay. This double-counted overtime and inflated the FICA base from $27,000 to $27,628."
 us,scenario_091,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used AGI near $36,183 and an estimated standard deduction near $10,668 instead of AGI of $35,012.56 and the $12,067.66 Wisconsin deduction. It then submitted $1,257 despite its own bracket calculation producing about $975."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly applied the federal American Opportunity Tax Credit against Wisconsin income tax. It also started from federal taxable income rather than subtracting Wisconsin's own $12,067.66 standard deduction and $700 exemption from Wisconsin AGI."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at about $36,183 and understated the Wisconsin standard deduction at about $9,500. The traced inputs leave $22,244.90 taxable, not roughly $25,983."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an estimated $7,700 standard deduction instead of Wisconsin's $12,067.66 deduction and consequently overstated taxable income by more than $5,500. Its unsupported post-calculation adjustment from $1,093 to $1,163 compounded the error."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model substituted approximate AGI and a roughly $9,000 standard deduction for the traced AGI of $35,012.56 and deduction of $12,067.66. That inflated taxable income from $22,244.90 to about $26,500."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated 2024 parameters and an invented linear phaseout formula rather than the 2026 Wisconsin parameters. It therefore used the wrong AGI, standard deduction, exemption treatment, and bracket thresholds instead of taxing $22,244.90 to $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at $36,183 and understated the standard deduction at about $11,340, producing taxable income of about $24,143 instead of $22,244.90. It also invented a nonrefundable Married/Single credit for this single filer."
-us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used 2025 inputs, including AGI of $34,383, a flat $13,230 deduction, and 3.54%/4.65% rates. The required 2026 computation uses AGI of $35,012.56, a $12,067.66 deduction, a $700 exemption, and the 2026 brackets."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model omitted taxable income components when it set AGI at $29,594. The traced AGI is $35,012.56, and after the Wisconsin deduction and exemption taxable income is $22,244.90 rather than $17,484."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer treated $27,000 as the relevant Wisconsin AGI and ignored the net effect of investment income and applicable adjustments. The traced computation uses $35,012.56 of AGI and $22,244.90 of taxable income."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The submitted $588 does not follow from the traced $35,012.56 AGI, $12,067.66 standard deduction, and $700 exemption. Those inputs produce $22,244.90 of Wisconsin taxable income and $843.66 of tax."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $29,594 and overstated the standard deduction at about $12,707. It consequently taxed only $16,887 instead of the traced $22,244.90."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model applied an unsupported 60% long-term capital-gain exclusion and an unsupported medical-expense credit. The Wisconsin computation instead reaches $22,244.90 of taxable income after the traced deduction and exemption, with tax of $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,The submitted value was $944 while its required final explanation stated value = 1018. That internal mismatch violated the answer contract and neither number carried the traced taxable income through to $843.66.
-us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the Wisconsin standard deduction to zero. At the traced AGI, the applicable deduction is $12,067.66, followed by the $700 exemption, leaving $22,244.90 taxable."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model asserted that unspecified Wisconsin treatment eliminated the tax despite $22,244.90 of taxable income. Applying the 2026 brackets to that amount leaves $843.66 before refundable credits."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model improperly used the federal American Opportunity Tax Credit facts to eliminate Wisconsin tax. That federal credit does not offset this Wisconsin liability, which remains $843.66 before refundable credits."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model made unsupported subtractions for overtime, health-insurance premiums, and capital gains, reducing Wisconsin income to $33,404. The traced AGI is $35,012.56, and the applicable Wisconsin deduction is $12,067.66 rather than $12,513.92."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's approximate $943 calculation did not use the traced $12,067.66 Wisconsin standard deduction and $22,244.90 taxable income. Applying the 2026 brackets to the traced taxable amount yields $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model introduced a capital-gain subtraction and rounded the tax rather than following the traced Wisconsin income calculation. The correct deduction and exemption leave $22,244.90 taxable, producing $843.66 rather than $847."
-us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model supplied only a rounded bracket estimate and did not carry the traced $22,244.90 taxable income through the 2026 Wisconsin schedule. That schedule produces $843.66, not $860."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The model invented credits that fully offset the Wisconsin liability. The standard deduction and exemption reduce income to $22,244.90 but do not eliminate it, leaving $843.66 before refundable credits."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model overstated AGI at $36,183 and understated the sliding-scale standard deduction, leaving about $24,437 taxable. The traced inputs leave $22,244.90 taxable and produce $843.66."
-us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model omitted taxable income components when it used Wisconsin AGI of $29,594. It also failed to subtract the $700 personal exemption, whereas the traced AGI, deduction, and exemption leave $22,244.90 taxable."
-us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used approximate AGI and deduction amounts and applied a 4.65% marginal rate instead of the traced 2026 Wisconsin schedule. The exact inputs produce $22,244.90 of taxable income and $843.66 of tax."
-us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. It therefore failed the required structured-output contract.
-us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used Wisconsin income of $35,832 and a standard deduction of $11,868.47 instead of the traced $35,012.56 AGI and $12,067.66 deduction. Those errors raised taxable income from $22,244.90 to $23,263.53."
-us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model applied the Wisconsin School Property Tax Credit even though no qualifying property-tax or rent input was listed and unlisted inputs were zero. That unsupported credit cannot eliminate the $843.66 pre-refundable-credit liability.
-us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model arbitrarily subtracted half of the employer-sponsored insurance premium from wages, used a federal-style flat standard deduction, and treated the $700 personal exemption as a $77 credit. Wisconsin instead uses traced AGI of $35,012.56, a $12,067.66 sliding-scale deduction, and a $700 exemption from income."
-us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model invented unspecified nonrefundable Wisconsin credits large enough to erase $1,374.94 of tax. No such offset applies here; the traced deduction and exemption leave $22,244.90 taxable and $843.66 due before refundable credits."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used AGI near $36,183 and an estimated $10,668 standard deduction instead of Wisconsin AGI of $35,012.56 and the $12,067.66 deduction. It also submitted $1,257 after its own revised bracket calculation produced about $975."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly applied the federal American Opportunity Tax Credit against Wisconsin income tax and treated Wisconsin as starting from federal taxable income. Wisconsin taxable income here is $22,244.90 after the state deduction and exemption, and no stated nonrefundable Wisconsin credit eliminates its $843.66 tax."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin AGI at about $36,183 and understated the Wisconsin standard deduction at about $9,500. The trace instead uses $35,012.56 of AGI and a $12,067.66 deduction, leaving $22,244.90 taxable."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model estimated the sliding Wisconsin standard deduction as only $7,700 and consequently taxed about $27,783. The applicable deduction is $12,067.66, and taxable income after the $700 exemption is $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used approximate AGI of $36,183 and a standard deduction near $9,000, overstating taxable income by more than $4,000. Wisconsin uses AGI of $35,012.56 and a $12,067.66 standard deduction."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model projected 2026 parameters from 2024 figures instead of applying the actual 2026 Wisconsin inputs. Its assumed $36,183 AGI and $9,688 deduction produced $25,795 taxable income rather than the traced $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model used $36,183 of income and an approximate $11,340 deduction rather than the traced Wisconsin AGI and deduction. It also invented a nonrefundable “Married/Single credit” reduction for this single filer instead of applying the tax schedule directly to $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model substituted 2025 rules, including 3.54% and 4.65% rates, for the requested 2026 Wisconsin calculation. It also used $34,383 of AGI and a flat $13,230 deduction instead of the traced $35,012.56 AGI and $12,067.66 sliding deduction."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model omitted enough reported investment income to reduce AGI to $29,594. Wisconsin AGI is $35,012.56, and subtracting the $12,067.66 standard deduction and $700 exemption produces $22,244.90 taxable income."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer treated the $27,000 wage figure as the relevant Wisconsin AGI and did not account specifically for the taxable investment income and state adjustments. The correct state computation starts from $35,012.56 and reaches $22,244.90 after deductions."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $588 result does not follow from the traced $35,012.56 Wisconsin AGI, $12,067.66 standard deduction, and $700 exemption. Those inputs leave $22,244.90 taxable and produce $843.66 under the 2026 brackets."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated Wisconsin AGI as $29,594 and overstated the standard deduction as $12,707. This reduced taxable income to $16,887 instead of the traced $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The model applied a 60% long-term-capital-gain exclusion and an unspecified medical-expense credit that are not part of this traced liability. The state computation instead reaches $22,244.90 of taxable income and has $843.66 remaining before refundable credits."
+us,scenario_091,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model submitted $944 while its explanation ended with $1,018, violating the requirement that the explanation's final amount exactly match the numeric output. Neither amount implements the traced $22,244.90 taxable-income calculation."
+us,scenario_091,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model declared the Wisconsin standard deduction fully phased out at this income. The applicable deduction is $12,067.66, which reduces taxable income to $22,244.90 after the $700 exemption."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model asserted that deductions and unspecified state treatment eliminate the liability. The Wisconsin deduction and exemption leave $22,244.90 taxable, so $843.66 remains before refundable credits."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model incorrectly used the federal American Opportunity Credit facts to eliminate Wisconsin tax. No applicable nonrefundable Wisconsin credit offsets the tax calculated on $22,244.90 of state taxable income."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted overtime and health-insurance amounts in constructing Wisconsin income and used an estimated deduction. The trace uses Wisconsin AGI of $35,012.56 and a $12,067.66 standard deduction, not its $33,404 income shortcut."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model gave only an approximation and did not apply the exact 2026 Wisconsin parameters. Taxing the traced $22,244.90 of taxable income yields $843.66, not $943."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model invoked a capital-gain subtraction without reproducing the traced Wisconsin AGI and deduction calculation. The exact inputs produce $22,244.90 taxable income and $843.66, rather than the rounded $847 estimate."
+us,scenario_091,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used a broad estimate instead of the exact Wisconsin deduction and bracket computation. Applying the 2026 schedule to $22,244.90 of taxable income gives $843.66, not $860."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model asserted that unspecified Wisconsin credits and the standard deduction fully offset the tax. After the $12,067.66 deduction and $700 exemption, $22,244.90 remains taxable and no stated nonrefundable credit reduces the resulting liability to zero."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model started from $36,183 and derived about $24,437 of taxable income using approximate inflation adjustments. The trace starts from $35,012.56 and yields $22,244.90 after the exact $12,067.66 deduction and $700 exemption."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $10,618 standard deduction after its own capital-gain adjustment. The applicable Wisconsin deduction is $12,067.66, leaving $22,244.90 rather than about $24,514 taxable."
+us,scenario_091,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model omitted reported investment income or otherwise understated Wisconsin AGI as $29,594. The correct Wisconsin AGI is $35,012.56, and the exact deduction and exemption leave $22,244.90 taxable."
+us,scenario_091,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used approximate deductions and a 4.65% second-bracket rate rather than the exact 2026 Wisconsin computation. The trace applies the governing schedule to $22,244.90 and produces $843.66."
+us,scenario_091,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so it failed the required structured-output contract."
+us,scenario_091,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model calculated an $11,868.47 standard deduction after a capital-gain exclusion, rather than using the traced $12,067.66 deduction and $35,012.56 AGI. Its taxable income of $23,263.53 is therefore $1,018.63 too high."
+us,scenario_091,state_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model invented a fully offsetting Wisconsin School Property Tax Credit despite the prompt listing no rent or property-tax input. The $843.66 liability calculated from $22,244.90 of taxable income therefore remains before refundable credits."
+us,scenario_091,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model estimated a $14,480 base deduction and used the wrong phaseout threshold, producing a deduction near $13,480. The traced income-sensitive deduction is $12,067.66, and the resulting taxable income is $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model arbitrarily deducted half of the employer-sponsored insurance premium from wages, treated the $700 personal exemption as a $77 credit, and cycled through unsupported standard deductions. Wisconsin instead uses $35,012.56 of AGI, subtracts $12,067.66 and $700, and taxes $22,244.90."
+us,scenario_091,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model invented unspecified nonrefundable Wisconsin credits sufficient to erase $1,374.94 of calculated tax. It also used $40,571 of income and a flat $12,200 deduction instead of the traced $35,012.56 AGI and $12,067.66 income-sensitive deduction."
 us,scenario_091,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_092,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the full $17,400 Social Security benefit as gross income instead of applying the provisional-income formula, under which none of it is taxable. With AGI of $14,980, the aged-single standard deduction eliminates all taxable income."
 us,scenario_092,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly included 85% of Social Security by comparing total income directly with the $34,000 threshold; Social Security taxation uses provisional income, including only one-half of benefits, and yields no taxable Social Security here. It also inconsistently applied the basic and age-based standard-deduction amounts, but the correct $14,980 AGI is fully offset by the applicable deduction."
@@ -5511,55 +5788,59 @@ us,scenario_092,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibilit
 us,scenario_092,head_medicaid_eligible,qwen-3.7-max,llm_error,thresholds_rates,False,"The model applied an approximately $2,829 monthly SSI-related income limit that does not establish eligibility for this Alabama case, then compared income after a $20 disregard to that incorrect standard. The person receives no SSI and qualifies through no aged/disabled category, so that threshold calculation cannot produce Medicaid eligibility."
 us,scenario_092,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly recognized that the household has no wages, then improperly invented an “implied Social Security-taxable earnings equivalent” equal to the $17,400 Social Security retirement benefit. Retirement benefits are not employee payroll-taxable earnings, so applying the combined 7.65% FICA rate to that amount produced the erroneous $1,253.76 instead of $0."
 us,scenario_092,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,age_disability,False,"The model treated the pension as taxable without applying Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older. It also used the standard deduction instead of the larger $9,028 itemized deduction, whereas the exclusion followed by itemization eliminates taxable income."
-us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,age_disability,False,"The model correctly itemized $9,028 but started from the full $14,980 pension because it omitted Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older. The exclusion reduces income to $8,980, below the itemized deductions, leaving no taxable income."
-us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,age_disability,False,"The model omitted the $6,000 Alabama retirement-income exclusion available at age 65 or older and selected a $3,000 standard deduction despite $9,028 of itemized deductions. Applying the exclusion and the larger itemized deduction reduces taxable income to zero."
-us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,age_disability,False,"The model correctly excluded Social Security and calculated the itemized deductions, but failed to exclude $6,000 of pension income under Alabama's age-65 retirement-income rule. The remaining $8,980 is less than the $9,028 itemized deduction, so none reaches the tax brackets."
-us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,age_disability,False,"The model omitted Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older. After that exclusion, the $9,028 itemized deduction exceeds the remaining $8,980 of pension income, producing zero taxable income."
-us,scenario_092,state_income_tax_before_refundable_credits,glm-5.2,llm_error,age_disability,False,"The model substituted a $1,000 age exemption for Alabama's separate $6,000 retirement-income exclusion available to taxpayers age 65 or older. It also chose the $3,000 standard deduction instead of $9,028 of itemized deductions; the correct combination eliminates taxable income."
-us,scenario_092,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,age_disability,False,"The model reduced the pension only by a standard deduction and exemptions, omitting Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older and the larger $9,028 itemized deduction. Those two steps reduce taxable income to zero."
-us,scenario_092,state_income_tax_before_refundable_credits,grok-4.5,llm_error,age_disability,False,"The model correctly excluded Social Security and used the larger itemized deduction, but omitted Alabama's $6,000 age-65 retirement-income exclusion. That exclusion lowers the pension amount to $8,980, which is fully offset by $9,028 of itemized deductions."
-us,scenario_092,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The $182 result is consistent with taxing the full $14,980 pension after itemized deductions and a personal exemption while omitting Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older. Applying the exclusion before the $9,028 itemized deduction leaves zero taxable income."
-us,scenario_092,state_income_tax_before_refundable_credits,inkling,llm_error,age_disability,False,"The model correctly calculated $9,028 of itemized deductions but failed to apply Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older. The remaining $8,980 of pension income is less than those deductions, so taxable income is zero."
-us,scenario_092,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, violating the required output contract."
-us,scenario_092,state_income_tax_before_refundable_credits,kimi-k3,llm_error,age_disability,False,"The model omitted Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older and used a $2,500 standard deduction instead of the larger $9,028 itemized deduction. Applying the exclusion and itemization leaves no taxable income."
-us,scenario_092,state_income_tax_before_refundable_credits,minimax-m3,llm_error,age_disability,False,"The model used an erroneous $29,770 Alabama AGI instead of excluding Social Security and applying Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older to the pension. It also omitted the $9,028 itemized deduction; the correct income exclusion and deductions reduce taxable income to zero."
-us,scenario_092,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,age_disability,False,"The model invented age-based standard-deduction amounts instead of applying Alabama's $6,000 retirement-income exclusion for taxpayers age 65 or older and ignored the larger $9,028 itemized deduction. It then made an internally invalid adjustment that raised its stated $349 bracket tax to $852.45 despite describing a federal-tax deduction, which would reduce rather than increase taxable income."
+us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model treated the private pension as Alabama taxable income and merely reduced it by the standard deduction. Alabama's pension exclusion removes the qualifying $14,980 pension from taxable income, leaving no liability."
+us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but incorrectly retained the entire $14,980 private pension in Alabama AGI. That pension is excluded under Alabama's retirement-income rules, so there is no $4,452 taxable balance to which the brackets apply."
+us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model taxed the $14,980 pension after subtracting a standard deduction and exemptions. Alabama excludes the qualifying pension itself, leaving zero taxable income rather than $9,480."
+us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security but treated the $14,980 private pension as Alabama gross income before itemized deductions. Alabama's pension exclusion removes that income, so its $4,452 taxable-income calculation and $182.60 bracket tax do not arise."
+us,scenario_092,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly made the $14,980 pension taxable in Alabama and offset it only with itemized deductions and the personal exemption. Excluding the qualifying pension leaves no Alabama taxable income and no bracket liability."
+us,scenario_092,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model correctly removed Social Security but incorrectly retained the private pension in Alabama AGI. The qualifying pension is excluded, so the standard deduction and age and personal exemptions do not leave $9,480 of taxable income."
+us,scenario_092,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated the private pension as taxable and attempted to reduce it through deductions and exemptions. Alabama excludes the qualifying pension at the income stage, producing no taxable income and no state bracket tax."
+us,scenario_092,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly used the $14,980 pension as Alabama AGI and then itemized deductions against it. Alabama's qualifying-pension exclusion eliminates that income, so the asserted $4,452 taxable balance is zero."
+us,scenario_092,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model retained the entire private pension in Alabama AGI and tried to eliminate it only through itemized, personal, and age-related deductions. The pension exclusion removes the $14,980 before those calculations, leaving no $2,952 taxable balance."
+us,scenario_092,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model's $182 estimate implies that it taxed a residual portion of the private pension after deductions. Alabama excludes the qualifying pension itself, so the retirement-income calculation yields zero taxable income and zero tax."
+us,scenario_092,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but incorrectly treated the $14,980 pension as Alabama AGI. The qualifying pension is also excluded, so itemized deductions do not leave $4,452 subject to Alabama's rates."
+us,scenario_092,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_092,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model taxed the $14,980 pension after only a standard deduction and personal exemption. Alabama excludes the qualifying pension from taxable income, so the claimed $10,980 tax base does not exist."
+us,scenario_092,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used $29,770 as Alabama AGI, thereby including retirement income that Alabama excludes and also failing to remove Social Security fully. Excluding both Social Security and the qualifying private pension leaves no taxable retirement income rather than $24,770 after exemptions."
+us,scenario_092,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly taxed the private pension and focused on a phased-down standard deduction and an incorrect top marginal rate. Alabama's qualifying-pension exclusion removes the $14,980 from the tax base entirely, leaving no $11,730 taxable balance."
+us,scenario_092,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly retained the private pension in Alabama taxable income and then introduced unsupported approximate deduction adjustments that contradict its own $349 bracket calculation. Alabama excludes the qualifying pension, and a federal-tax deduction cannot turn that zero tax base into $852.45 of liability."
 us,scenario_092,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_093,dependent1_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated the disability flag as sufficient for Missouri's SSI-related or MO HealthNet for the Disabled pathway and explicitly disregarded the pathway's additional eligibility requirements. Dependent 1 receives no SSI and qualifies for no Medicaid category, while their 2.82-times-FPL MAGI also disqualifies them from the MAGI pathways."
 us,scenario_093,dependent1_medicare_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model treated the listed disability flag as automatic Medicare eligibility and explicitly assumed completion of an SSDI waiting period despite the instruction that unlisted statuses are false. No SSDI receipt, 24-month entitlement period, ESRD, or ALS was listed, so the dependent does not meet an under-65 Medicare pathway."
 us,scenario_093,dependent1_medicare_eligible,claude-sonnet-4.6,llm_error,health_coverage,False,"The model correctly identified the SSDI-based 24-month rule but then replaced its required SSDI entitlement history with the generic disability flag. PolicyEngine does not equate disability status with Medicare eligibility, and the unlisted SSDI, ESRD, and ALS criteria are false."
 us,scenario_093,dependent1_medicare_eligible,glm-5.2,llm_error,health_coverage,False,"The model misstated PolicyEngine's rule as age 65 or disability status. For someone age 27, the disability flag alone is insufficient; none of the required under-65 Medicare entitlement criteria was supplied."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,000 of personal auto-loan interest and then abandoned its own resulting $8,509 calculation for an unsupported $8,067 estimate. The applicable taxable income is $80,208.25, and the resulting $9,128.99 liability is reduced by the $500 dependent credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model included the adult dependent's $45,000 wages in the couple's joint return and used an obsolete $29,200 standard deduction. It also subtracted an invented $4,246 of refundable credits from an output defined before refundable credits, instead of applying only the $500 nonrefundable dependent credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model rejected the disabled adult as a dependent by inferring that $45,000 of wages established self-support, despite the prompt's dependent designation and the engine's qualifying-child treatment. It therefore omitted the $500 nonrefundable dependent credit and also used incorrect projected deduction and bracket parameters."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the disabled adult's gross income as disqualifying and omitted the $500 nonrefundable dependent credit. It also used a $31,500 standard deduction rather than $32,200, overstating taxable income."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $3,000 of student-loan interest instead of the $2,500 cap and used an incorrect $32,600 standard deduction. Its final $10,371 contradicts its own approximately $8,371 computation and has no supporting tax step."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a $30,000 projected standard deduction instead of the 2026 $32,200 amount and used incorrect projected bracket thresholds. It also denied the $500 credit by applying the qualifying-relative gross-income test instead of the disabled qualifying-child pathway represented by the household facts."
-us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an estimated $30,300 standard deduction instead of $32,200 and omitted the $500 nonrefundable dependent credit. It then replaced its own $9,376 calculation with an unsupported $10,800 figure."
-us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model summed the couple's joint tax and a separate single-return tax for the dependent, but this output is the tax unit's joint federal income tax measure. It also treated listed employer premiums as additional pre-tax deductions even though the trace's wage computation reduces employment income only by $4,058.12 of traditional 401(k) contributions."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model assumed a 2026 TCJA sunset with personal exemptions and reduced standard deductions, contrary to the applicable 2026 parameters. It also added a separate return for the dependent and deducted listed ESI premiums from wages, neither of which belongs in this output's derivation."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model used all $163,000 of household wages as income on one married joint return, improperly including the adult dependent's $45,000 wages. The joint tax calculation uses only the head's and spouse's income, producing $112,408.25 of AGI before the standard deduction."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a nonexistent 2026 reversion to pre-TCJA parameters and combined the couple's liability with a separate liability for the dependent. The applicable calculation uses the $32,200 joint standard deduction and the current 2026 joint brackets, followed by the $500 nonrefundable credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model added the disabled adult's separate single-return liability to the requested married tax-unit output. It also reduced wages by ESI premiums and applied personal exemptions, causing both component liabilities to depart from the traced computation."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model describes summing two tax units, while the requested value is the married tax unit's liability after its $500 nonrefundable credit. Its $9,127.42 is also nearly the couple's $9,128.99 pre-credit liability, showing that the claimed $500 credit and separate-return aggregation were not coherently applied."
-us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model summed a married joint return and a separate single return for the dependent. The requested measure uses the couple's $80,208.25 taxable income and subtracts the $500 nonrefundable dependent credit, without adding the dependent's own tax."
-us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added $3,353.74 of separate tax for the dependent to the couple's requested tax-unit output. For the couple it also used an estimated $30,840 standard deduction and incorrect thresholds instead of the $32,200 deduction and traced 2026 schedule."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that retirement deductions and the standard deduction eliminate taxable income. They leave $80,208.25 taxable, which generates $9,128.99 before the $500 nonrefundable credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model's $6,347 estimate does not follow from the stated wages, allowed adjustments, and standard deduction. The correct sequence produces $112,408.25 of AGI and $80,208.25 of taxable income, and the disabled adult supports a $500 nonrefundable credit rather than no dependent credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the dependent's wages in joint income while subtracting ESI premiums from all listed wages, producing an erroneous $132,241 AGI. The married return contains only head and spouse income, and listed employment income is reduced by $4,058.12 of pre-tax 401(k) contributions, not by those ESI amounts."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model's $124,708 taxable income reflects inclusion of income outside the married couple's return and is $44,499.75 above the traced taxable income. It also omitted the $500 nonrefundable credit for the disabled adult dependent."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The $16,511.56 result is consistent with taxing household-level wages rather than restricting the joint return to the head and spouse. The correct joint AGI is $112,408.25, after which the $32,200 standard deduction and $500 credit yield $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income by exactly $5,000, using $85,208 instead of $80,208.25. The traced $112,408.25 AGI less the $32,200 standard deduction fixes that error before applying the $500 dependent credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The $13,980 estimate is inconsistent with the couple's traced $112,408.25 AGI and implies that additional household income was taxed or valid deductions were omitted. The joint calculation reaches $9,128.99 before the $500 nonrefundable credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a post-TCJA-sunset regime with personal exemptions and then added a separate single-return tax for the dependent. The applicable 2026 computation instead uses the $32,200 joint standard deduction and subtracts the $500 nonrefundable dependent credit from the couple's $9,128.99 liability."
-us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model added $3,820 of separate tax for the dependent to the requested joint tax-unit result. It also used a $30,750 deduction and post-TCJA bracket assumptions rather than the traced $32,200 standard deduction and applicable 2026 rates, and omitted the $500 credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model correctly approximated the couple's $9,129 pre-credit liability but then added roughly $3,220 of separate tax for the dependent. It also failed to subtract the $500 nonrefundable dependent credit, which converts $9,128.99 to $8,628.99."
-us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for the requested output, so the required structured result was missing."
-us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the dependent's $45,000 wages in the parents' married joint return. It also failed to subtract the $4,058.12 of pre-tax 401(k) contributions and wrongly denied the $190 traditional IRA deduction, inflating AGI far above $112,408.25."
-us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer disregards the positive taxable income left after allowed adjustments and the standard deduction. The computation leaves $80,208.25 taxable and $8,628.99 due after the $500 nonrefundable credit."
-us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted listed ESI premiums from wages, used the full $3,000 student-loan-interest amount instead of the $2,500 cap, and denied the dependent credit. Its submitted $7,625.75 also does not match any of its own calculated alternatives, including $7,248.40 and approximately $9,357."
-us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model included all $163,000 of household wages in one married return and used incorrect deduction inputs, yet its submitted $3,778 does not match its own stated $13,491 liability. It also incorrectly asserted that the $500 nonrefundable dependent credit is excluded from an output expressly defined after nonrefundable credits."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,000 of personal auto-loan interest, reducing taxable income from $80,208.25 to $79,208.25. It then abandoned its own $8,509 computation for an unsupported $8,067 estimate instead of applying the traced 2026 parameters and $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model included the adult dependent's $45,000 wages in the parents' joint income, even though those wages are not part of this joint tax unit's gross income. It also used the wrong standard deduction and subtracted an invented $4,246 reconciliation amount."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model rejected the adult disabled dependent and therefore omitted the $500 nonrefundable dependent credit. It also used a $31,500 standard deduction instead of $32,200, leaving taxable income $700 too high."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated the dependent's wages as disqualifying and omitted the $500 nonrefundable credit. It also used an understated $31,500 standard deduction and approximate bracket thresholds rather than taxing $80,208.25 under the traced 2026 schedule."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model first derived a liability near $8,371, then added an unsupported $2,000 for the disabled adult's supposed separate tax. The requested output is the joint tax unit's liability after its $500 nonrefundable credit, not a household sum augmented by another return."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used a projected $30,000 standard deduction instead of the 2026 $32,200 amount, overstating taxable income by $2,200. It also denied the $500 dependent credit, so both its pre-credit tax base and credit calculation were too high."
+us,scenario_093,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly reached AGI near $112,408 but used an estimated $30,300 standard deduction and then replaced its own $9,376 calculation with an unexplained $10,800. It also omitted the $500 nonrefundable dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model summed a joint return and a separate return for the adult dependent. The requested value is the joint tax unit's $9,128.99 pre-credit liability less its $500 nonrefundable dependent credit, with no separate dependent-return tax added."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,The model incorrectly applied a post-TCJA-sunset regime with personal exemptions and reduced standard deductions. It also added a separate return for the dependent instead of using the traced 2026 joint computation and its $500 credit.
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model based its calculation on all $163,000 of household wages, thereby including the dependent's $45,000 wages in the joint tax base. The joint return instead starts from the head's and spouse's wages and reaches $112,408.25 AGI."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied an obsolete assumed reversion to pre-TCJA parameters and combined liabilities from joint and separate returns. The traced 2026 rules use a $32,200 joint standard deduction and a $500 nonrefundable dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model computed and added a separate single-return liability for the dependent. It also reduced wages by employer-sponsored premiums and used personal exemptions, neither of which belongs in the traced joint-tax calculation."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model added tax from a separate return for the adult dependent to the requested joint-tax output. The correct joint computation alone produces $9,128.99 before credits and $8,628.99 after the $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The answer explicitly sums an MFJ tax unit and a separate single tax unit for the dependent. The requested variable is the joint tax unit's federal income tax after its nonrefundable credit, not the aggregate of two returns."
+us,scenario_093,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added $3,353.74 of separate-return tax for the dependent to the parents' liability. It also used a $30,840 joint standard deduction instead of $32,200; the requested joint result excludes that separate return and equals $8,628.99 after the $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated $112,408.25 of AGI as fully sheltered by deductions and credits. The $32,200 standard deduction leaves $80,208.25 taxable, producing $9,128.99 before the $500 credit rather than zero."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model's $6,347 estimate reflects excessive deductions or exclusions not supported by the inputs. The allowable adjustments produce $112,408.25 AGI and the standard deduction leaves $80,208.25 taxable; it also wrongly stated that the dependent credit was inapplicable."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included part of the adult dependent's wages in joint box-1 wages and separately subtracted employer-sponsored premiums from stated wages. Joint gross income instead comprises $113,942 after traditional 401(k) contributions plus $1,156 interest, leading to $112,408.25 AGI."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 taxable-income estimate necessarily includes income outside the joint tax unit, chiefly the dependent's wages. The correct taxable income is $80,208.25, and the model also omitted the $500 nonrefundable dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The stated inputs and $500 credit cannot yield $16,511.56 unless the dependent's wages are placed in the joint tax base or equivalent excess income is added. The correct joint taxable income is $80,208.25 before applying the credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income by exactly $5,000, using $85,208 instead of $80,208.25. The traced $112,408.25 AGI less the $32,200 standard deduction supplies the correct tax base before the $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The $13,980 estimate is consistent with taxing income beyond the head-and-spouse joint return, including the dependent's wages or an added separate liability. The joint computation taxes $80,208.25 and then subtracts the $500 dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model imposed a post-TCJA-sunset system with personal exemptions and then added the dependent's separate-return tax. The applicable 2026 computation instead uses the $32,200 joint standard deduction, current brackets, and the $500 nonrefundable dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used pre-TCJA-style 10% and 15% brackets, personal exemptions, and a $16,600 joint standard deduction. It then added a separate dependent return and omitted the $500 credit, whereas the traced 2026 joint liability uses $80,208.25 of taxable income."
+us,scenario_093,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model added $3,820 of tax from a separate dependent return to the requested output. Its joint component also used the wrong standard deduction and tax schedule and omitted the $500 nonrefundable dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model correctly approximated the joint pre-credit tax at $9,129 but then added roughly $3,220 from a separate dependent return. It also failed to subtract the joint tax unit's $500 nonrefundable dependent credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_093,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the dependent's $45,000 wages in the parents' married-joint return. It also omitted the $4,058 traditional 401(k) exclusion and $190 IRA deduction, so its AGI and taxable income were substantially overstated."
+us,scenario_093,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The zero answer ignores the positive taxable income remaining after allowable deductions. AGI of $112,408.25 less the $32,200 standard deduction leaves $80,208.25 taxable, and the $500 credit does not eliminate the resulting liability."
+us,scenario_093,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model added the dependent's $45,000 wages to the married-joint return. Excluding those wages yields the traced $112,408.25 AGI and $80,208.25 taxable income before applying the $500 credit."
+us,scenario_093,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model subtracted $16,778 of employer-sponsored premiums from the head's and spouse's stated wages and deducted $3,000 of student-loan interest rather than the $2,500 cap. It then returned $7,625.75 despite deriving several different totals, and it omitted the $500 dependent credit from its selected joint-only calculation."
+us,scenario_093,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model's submitted $3,778 does not follow from its own stated regular liability of about $13,491 and no credits. It also included the dependent's wages in joint income and incorrectly claimed that the requested after-nonrefundable-credit metric does not subtract the $500 dependent credit."
 us,scenario_093,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_093,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly listed all six employee FICA components, which sum to $12,469.50, but then stated an intermediate total of $11,469.50 and submitted the unrelated value $12,348.76. Its submitted output does not follow its own component calculation."
 us,scenario_093,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model explicitly derived $10,106 of Social Security tax plus $2,363.50 of Medicare tax, totaling $12,469.50, but submitted $12,487.50. It introduced an unexplained $18 transposition error after completing the calculation correctly."
@@ -5574,36 +5855,38 @@ us,scenario_093,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model
 us,scenario_093,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll-tax value or explanation, so it failed the required output contract."
 us,scenario_093,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer omits employee Social Security and Medicare taxes on all three workers' $163,000 of wages. Those wages generate $10,106 of Social Security tax and $2,363.50 of Medicare tax."
 us,scenario_093,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model correctly calculated person-level FICA amounts of $4,054.50, $4,972.50, and $3,442.50, which sum to $12,469.50, but submitted $12,553.50. It added an unexplained $84 after deriving the correct total."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached approximately the correct $79,800 taxable-income base but used an approximate 4.7% schedule and bracket adjustment instead of the enacted Missouri schedule. Applying the exact brackets to $79,776.80 yields $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model started from $119,156 rather than the traced $115,097.88 of IRS gross income, omitted the $2,689.62 above-the-line deductions and Missouri deduction computation, and substituted an effective-rate estimate. It also discussed separate spouse and dependent tax amounts even though this output is the joint filers' Missouri tax unit."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model failed to reduce gross income to the traced $115,097.88 and used only an estimated standard deduction, leaving taxable income at $86,466 instead of $79,776.80. It then applied an approximate rate calculation rather than the exact Missouri brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its stated taxable income of about $80,408 is close to the traced $79,776.80, but its arithmetic is internally wrong: the formula it displays is about $3,640, not $4,126. The exact Missouri schedule on the traced base produces $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model added roughly $1,400 of tax for the 27-year-old to the joint return. PolicyEngine's requested output assigns the Missouri liability only across the two primary earners, while the third household member is exempt from Missouri income tax in this calculation."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly treated the listed employer-sponsored insurance premiums as pretax wage reductions, reducing income by $16,778 without facts establishing a Section 125 exclusion. It also invented a $4,400 Missouri personal exemption and deducted the full estimated federal tax rather than using the traced Missouri deductions, driving taxable income far below $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted taxable income of roughly $102,000 despite also invoking the joint standard deduction and other reductions. The traced sequence produces only $79,776.80 of Missouri taxable income, so its $4,800 estimate taxes an overstated base."
-us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model added a separate $1,056.06 single-filer liability for the 27-year-old. That person is exempt from Missouri income tax in this output, and the applicable joint-unit liability is calculated under 2026 law rather than 2024 brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an estimated 4.8% top rate instead of the applicable 2026 Missouri schedule and did not reproduce the traced $79,776.80 taxable-income base. Its $4,597.60 therefore reflects both an obsolete rate assumption and an overstated tax base."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The unsupported $6,055 answer does not implement the traced computation. Missouri taxable income is $79,776.80, and the applicable brackets produce $3,389.65 rather than the submitted amount."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model replaced the Missouri bracket calculation with a rough 4.5% estimate on vaguely defined combined income. Applying the exact schedule to the traced $79,776.80 taxable-income base yields $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although the model named the relevant deduction categories, its $4,166 answer does not reflect the traced $12,469.50 Missouri deduction or the resulting $79,776.80 taxable income. The exact Missouri schedule on that base produces $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model supplied no computation supporting its $3,421 figure. The traced deductions produce $79,776.80 of taxable income, and exact application of the Missouri schedule gives $3,389.65 rather than a rounded approximation."
-us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model gave no usable derivation and its $4,850 answer does not follow from the traced tax base. Taxing $79,776.80 under the Missouri schedule yields $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the dependent's $30,400 of purported taxable income to the joint filers' base and charged tax on both units. The third household member is exempt from Missouri income tax in this output, and Missouri also uses graduated brackets rather than a flat 4.7% multiplication."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed deductions and allowances eliminate all Missouri taxable income. After the traced adjustments and deductions, $79,776.80 remains taxable and produces $3,389.65 of tax."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model offered only a generic approximation and did not calculate the traced taxable-income base or exact brackets. The prescribed computation yields $3,389.65, not $3,427."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the 27-year-old's $45,000 wages in a $157,598 federal AGI for the joint return, even though those wages are outside the joint tax unit. It also invented Missouri exemptions and a health-insurance subtraction, whereas the traced joint taxable income is $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 taxable-income estimate necessarily incorporates income outside the joint unit or omits major deductions. The correct joint-unit sequence leaves $79,776.80 taxable and excludes the exempt third household member's Missouri tax."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model reduced an unspecified federal AGI only by the Missouri standard deduction and incorrectly discarded the federal-income-tax deduction based on an asserted phaseout. The trace includes $431.45 for federal income taxes paid within $12,469.50 of Missouri deductions and produces $79,776.80 taxable income."
-us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model invoked exemptions and a federal-income-tax deduction without matching the traced Missouri deduction calculation. Those traced deductions leave $79,776.80 taxable, whose exact scheduled tax is $3,389.65 rather than $3,192.44."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an unsupported approximation after the standard deduction rather than the exact Missouri schedule. The schedule applied to $79,776.80 produces $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model separately taxed the dependent and added $1,547 to the household result. The third household member is exempt from Missouri income tax in this output, and the joint filers' traced taxable income is $79,776.80 rather than about $95,365."
-us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model improperly added $1,354 of Missouri tax for the dependent and used a 4.95% schedule. The output contains only the liability distributed across the two joint filers, calculated under the applicable 2026 brackets."
-us,scenario_093,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model added about $1,047 for a separate single tax unit, but the third household member is exempt from Missouri income tax in this calculation. It also understated the joint-unit component relative to the exact $3,389.65 bracket result."
-us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model used invented Missouri parameters: a $13,000 joint standard deduction, $6,000 of personal exemptions, a 35% federal-tax formula, and an obsolete multi-rate schedule reaching 5.3%. The trace instead starts from $112,408.25 of Missouri AGI, subtracts $12,469.50 under the applicable deduction computation, and taxes $79,776.80 under the 2026 schedule."
-us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model treated all $161,656 of household income, including the 27-year-old's wages, as joint federal AGI. The joint return's traced IRS gross income is $115,097.88, and the third household member is exempt from Missouri income tax in this output."
-us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,missing_output,False,"The model provided no reasoning and asserted zero tax. The traced computation leaves $79,776.80 of taxable income and produces $3,389.65."
-us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the dependent's $45,000 wages in joint AGI and taxed an overstated $137,556 base. It then contradicted its own $6,809 multiplication by submitting $11,145.20; the third household member is exempt and the joint taxable base is $79,776.80."
-us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model began from an unsupported $124,942 of Missouri taxable income and omitted the traced adjustments and Missouri deductions that reduce the base to $79,776.80. Applying the exact schedule to the correct base gives $3,389.65."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model reached taxable income near the correct $79,776.80 but applied an approximate 4.7% schedule and bracket adjustment instead of the exact Missouri brackets. Applying the statutory schedule to that base yields $3,389.65, not $3,506."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used $119,156 as AGI, a $27,900 deduction, and an invented 4.04% effective rate instead of deriving Missouri AGI of $112,408.25 and taxable income of $79,776.80. It also inconsistently discussed separate spouse and dependent liabilities while submitting only its joint-return estimate."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model started from an overstated $116,466 AGI because it failed to reproduce the engine's $2,689.62 of above-the-line deductions and then used an approximate $30,000 deduction. The required Missouri calculation uses $112,408.25 of AGI and $79,776.80 of taxable income."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"Its stated taxable-income estimate of $80,408 is close to the traced $79,776.80, but its arithmetic for the Missouri rate schedule is wrong: the formula it wrote does not equal $4,126. The exact brackets on $79,776.80 produce $3,389.65."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model improperly added roughly $1,400 of Missouri tax for the disabled 27-year-old's supposed separate return. The benchmark calculation assigns the third household member no Missouri income tax, so only the head-and-spouse joint liability belongs in the output."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly treated the listed employer-sponsored insurance premiums as pretax wage reductions, driving AGI down to $95,130, and then deducted fabricated Missouri personal exemptions and the full federal tax amount. The trace instead has Missouri AGI of $112,408.25 and only a $431.45 Missouri deduction for federal income tax."
+us,scenario_093,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted Missouri taxable income of about $102,000 without reconciling that figure to its own listed deductions. The traced deductions reduce Missouri AGI of $112,408.25 to taxable income of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model added $1,056.06 for a separate single-filer return by the third household member. That member is exempt from Missouri income tax in this calculation, and the model also used obsolete 2024 parameters instead of the 2026 schedule."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The amount reflects taxation beyond the joint head-and-spouse liability, consistent with including a separate single-filer calculation for the third adult. The third member contributes no Missouri tax here, and the joint return is calculated from $79,776.80 of taxable income."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $6,055 result is inconsistent with the traced Missouri taxable income of $79,776.80 and the applicable rate schedule. It reflects taxation of substantially more income than the joint return contains."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model replaced Missouri's bracket calculation with a rough 4.5% estimate on an unspecified combined base. The exact schedule applied to $79,776.80 yields $3,389.65."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although it named the relevant deduction categories, it did not use the traced amounts: Missouri AGI is $112,408.25, deductions reduce taxable income to $79,776.80, and the federal-income-tax deduction is only $431.45. Its $4,166 result therefore rests on an overstated tax base or deduction error."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's near miss reflects an imprecise standard/federal-tax deduction computation rather than the traced Missouri deduction calculation. Using $12,469.50 of Missouri itemized deductions and taxable income of $79,776.80 produces $3,389.65."
+us,scenario_093,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The unsupported $4,850 amount taxes more income than the head-and-spouse joint Missouri return contains, consistent with incorporating the third adult's wages or liability. The third member is exempt from Missouri tax in this computation."
+us,scenario_093,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model explicitly added a $30,400 taxable-income base for the disabled adult's separate single return, but that member has no Missouri tax in the benchmark calculation. It also treated Missouri as a flat 4.7% tax instead of applying the graduated schedule."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The standard deduction and other allowable deductions do not eliminate Missouri income. They leave $79,776.80 taxable, on which the applicable brackets produce $3,389.65 before refundable credits."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model used a generic approximation and did not calculate the traced $12,469.50 Missouri deduction amount, including the $431.45 federal-tax component. The exact resulting taxable income and brackets give $3,389.65 rather than $3,427."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model formed federal AGI of $157,598 by including the disabled adult's $45,000 wages in the couple's joint return. The joint Missouri calculation starts from IRS gross income of $115,097.88, while the third member contributes no Missouri liability."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"Its $124,708 taxable-income estimate includes income outside the head-and-spouse joint Missouri base or omits major deductions. The traced joint taxable income is $79,776.80, and the third member is exempt from Missouri tax."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model taxed an overstated federal-AGI base and incorrectly asserted that no federal-tax deduction applies. The joint return receives a $431.45 federal-income-tax deduction, and the third adult's wages do not enter its Missouri taxable income."
+us,scenario_093,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model understated tax by using unspecified exemptions and an overstated federal-income-tax deduction. The traced Missouri deductions include only $431.45 for federal tax and lead to $79,776.80 of taxable income."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used only an approximate standard-deduction calculation and did not reproduce Missouri's traced deduction composition. The correct base is $79,776.80 after $12,469.50 of Missouri itemized deductions."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model explicitly added $1,547 of tax for a separate return by the third adult, who is exempt from Missouri income tax in this calculation. Its couple base of about $95,365 also exceeds the traced joint taxable income of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"The model explicitly included $1,555 of Missouri tax for the third adult even though that person has no Missouri liability here. It also overstated the couple's taxable income as $95,808 instead of $79,776.80."
+us,scenario_093,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model added $1,354 for the third adult's separate return, contrary to the benchmark's zero Missouri tax for that member. It also used a 4.95% top rate and omitted Missouri modifications beyond the standard deduction."
+us,scenario_093,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model added about $1,047 of separate-filer tax for the third household member, who is exempt from Missouri tax in this calculation. Its joint-unit estimate also does not reproduce the exact $79,776.80 taxable base and 2026 brackets."
+us,scenario_093,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model invented a $13,000 Missouri standard deduction, $6,000 of personal exemptions, and a percentage-based federal-tax deduction exceeding $3,000. The trace instead uses a $32,200 standard deduction within the Missouri deduction calculation and only $431.45 for federal income taxes, producing $79,776.80 taxable income."
+us,scenario_093,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model incorrectly combined the third adult's $45,000 wages with the couple's income to create $161,656 of household federal AGI. The joint Missouri return begins with $115,097.88 of IRS gross income, and the third member contributes no Missouri tax."
+us,scenario_093,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer disregards the positive $79,776.80 Missouri taxable income remaining after all allowed deductions. The Missouri brackets impose $3,389.65 before refundable credits."
+us,scenario_093,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model's $157,408 AGI includes the third adult's wages in the couple's Missouri base and then applies unsupported personal exemptions. The third adult is exempt from Missouri tax here, and the joint return's Missouri AGI is $112,408.25."
+us,scenario_093,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model explicitly included the dependent adult's $45,000 wages in an MFJ AGI of $164,156, even though those wages do not belong on the head-and-spouse joint return. It then used obsolete deduction and rate figures and produced a final number inconsistent with its own $6,809 intermediate calculation."
+us,scenario_093,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model started from federal taxable income of roughly $124,942, far above the traced Missouri taxable income of $79,776.80. It failed to restrict the base to the joint tax unit and to apply the traced Missouri deduction calculation."
 us,scenario_095,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the full $14,022 of Social Security in AGI and then applied an age-65 single-filer standard deduction. Social Security is not taxable at this provisional-income level, and the surviving-spouse filing-status deduction eliminates the resulting $15,286.89 AGI."
 us,scenario_095,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model correctly reached approximately $15,286 of AGI but treated the head as a single filer and applied a $9,800 deduction. The surviving-spouse filing status provides the larger applicable standard deduction, reducing taxable income to zero."
 us,scenario_095,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly concluded that none of the Social Security is taxable, but then retained it in its $29,308 AGI and computed taxable income from that inflated figure. Excluding the nontaxable Social Security produces $15,286.89 of AGI, which the surviving-spouse standard deduction fully offsets."
@@ -5630,19 +5913,21 @@ us,scenario_095,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model i
 us,scenario_095,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an incorrect 200%-of-FPL broad-based categorical eligibility threshold and then invented a $95 monthly New Jersey minimum SNAP benefit. The household’s $29,605 income exceeds the applicable gross-income limit, and the minimum allotment applies only to an otherwise eligible household."
 us,scenario_095,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model labeled the household’s countable income “very low” without reconciling that claim with the listed $29,605 annual income. That income exceeds the applicable one-person gross-income limit, so the $500 asset balance cannot establish eligibility and the correct allotment calculation yields $0."
 us,scenario_095,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly treated $2,467 in monthly gross income as below New Jersey’s applicable broad-based categorical eligibility limit. Because the household fails the gross-income screen, it is ineligible and cannot receive the $24 monthly minimum allotment, which is reserved for otherwise eligible households."
-us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model treated the surviving spouse as a single filer and proceeded to tax $2,198 at 1.4%. A qualifying surviving spouse with $15,583 of New Jersey gross income is below New Jersey’s $20,000 filing threshold, so the liability is zero before exemptions or bracket calculations matter."
-us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model constructed taxable income from federal AGI and deductions, then applied the 1.4% bracket without enforcing the qualifying-surviving-spouse filing threshold. New Jersey gross income of $15,583 is below the applicable $20,000 threshold, producing zero tax."
-us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model correctly excluded Social Security and the retirement distribution but incorrectly taxed the residual self-employment income. It omitted the $20,000 New Jersey gross-income threshold for a qualifying surviving spouse, which eliminates the liability at $15,583 of gross income."
-us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied exemptions and the 1.4% bracket as though this filer were required to calculate tax on residual income. The surviving-spouse filing status carries a $20,000 New Jersey gross-income threshold, and $15,583 falls below it."
-us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model stopped after deriving $2,198 of income following the retirement exclusion and personal exemptions and taxed it at 1.4%. It failed to apply the qualifying-surviving-spouse $20,000 gross-income threshold, under which the household owes no New Jersey income tax."
-us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model calculated medical and personal deductions against residual self-employment income instead of first applying the filing-status threshold. As a qualifying surviving spouse with $15,583 of New Jersey gross income, the filer is below the $20,000 threshold and has zero liability."
-us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model applied retirement, personal, age, and medical deductions and then taxed the remainder. It omitted the qualifying-surviving-spouse $20,000 New Jersey gross-income threshold, which makes the tax zero at $15,583."
-us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model invoked a low-income tax reduction but still retained $18.46 of liability. The controlling rule is the $20,000 gross-income filing threshold for a qualifying surviving spouse, which reduces liability to zero because New Jersey gross income is $15,583."
-us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model taxed income remaining after the pension exclusion, medical deduction, and exemptions. It failed to apply the $20,000 New Jersey gross-income threshold for qualifying surviving spouses, under which $15,583 produces no tax."
-us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used federal AGI, only one exemption, and an incorrect 1.75% rate, but the prior controlling error was omitting the surviving-spouse filing threshold. New Jersey gross income of $15,583 is below the qualifying-surviving-spouse threshold of $20,000, so no bracket rate applies."
-us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model correctly identified major New Jersey exclusions and deductions but then applied the 1.4% bracket to approximately $2,010. It overlooked the $20,000 gross-income threshold attached to qualifying-surviving-spouse status, which yields zero tax at $15,583."
-us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, violating the required submission contract."
-us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,household_unit_or_filing_status,False,"The model deducted half of self-employment tax and a single $1,000 amount before applying the 1.4% bracket, bypassing the filing-status threshold. The filer is a qualifying surviving spouse whose $15,583 of New Jersey gross income is below the $20,000 threshold, so the liability is zero."
+us,scenario_095,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly excluded Social Security and the full 401(k) distribution but then taxed the remaining $2,198 after exemptions at 1.4%. It omitted New Jersey's low-income no-tax threshold, which reduces the liability on this amount to zero."
+us,scenario_095,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the age-qualified New Jersey retirement-income exclusion for the $11,385 401(k) distribution, leaving taxable income of $13,092.10. It also applied the 1.4% rate without the low-income no-tax threshold that yields zero liability after the proper exclusion."
+us,scenario_095,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model derived $2,009.66 after the retirement exclusion, exemptions, and medical deduction, then applied 1.4% directly. New Jersey's low-income no-tax threshold makes the tax on that remaining income zero."
+us,scenario_095,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly reduced income to $2,198 after the retirement exclusion and exemptions but treated the 1.4% bracket as creating tax from the first dollar. It omitted the low-income no-tax threshold that eliminates the liability."
+us,scenario_095,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model correctly calculated $2,198 after excluding the retirement distribution and deducting the personal and senior exemptions. It then incorrectly imposed the 1.4% rate without applying New Jersey's low-income no-tax threshold."
+us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,The model reduced the remaining self-employment income by exemptions and a medical deduction but then taxed the resulting low taxable income at 1.4%. The applicable low-income threshold eliminates the state income tax rather than leaving $26.
+us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model applied the bottom 1.4% rate to income remaining after the retirement, personal, age, and medical deductions. It failed to apply New Jersey's low-income no-tax threshold, which reduces the computed liability to zero."
+us,scenario_095,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model recognized a low-income tax reduction but applied it only partially, leaving $18.46. For this household's income after the retirement exclusion and exemptions, the low-income rule eliminates the entire liability."
+us,scenario_095,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model calculated $1,882 after the retirement exclusion, exemptions, and medical deduction and taxed it at 1.4%. It omitted the low-income no-tax threshold that makes the liability zero."
+us,scenario_095,state_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly identified the principal exclusions and deductions but imposed 1.4% on approximately $1,882. New Jersey's low-income threshold eliminates tax at that income level."
+us,scenario_095,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an incorrect 1.75% rate and deducted only one $1,000 exemption despite the taxpayer's age-65 exemption. More fundamentally, it failed to apply the low-income no-tax threshold, so the remaining income produces zero liability."
+us,scenario_095,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly arrived at approximately $2,010 after the retirement exclusion, exemptions, and medical deduction but then taxed it at 1.4%. The low-income no-tax threshold eliminates the liability on that amount."
+us,scenario_095,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so its response failed the required output contract."
+us,scenario_095,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model omitted the additional $1,000 age-65 exemption and treated half of self-employment tax as a New Jersey income deduction. It then applied 1.4% to $2,901 without the low-income no-tax threshold that makes the liability zero."
+us,scenario_095,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model limited the retirement exclusion to $10,000 instead of excluding the full $11,385 age-qualified distribution. It also taxed the remaining low income at 1.4% without applying New Jersey's no-tax threshold, which eliminates the liability."
 us,scenario_095,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly classified a $1,500 NJ ANCHOR homeowner property-tax benefit as a refundable state individual income tax credit. ANCHOR is outside `nj_refundable_credits`, which yields $0 for this household."
 us,scenario_095,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required `state_refundable_credits` output entirely. The required computation evaluates `nj_refundable_credits` at $0 and therefore yields a $0 statewide refundable-credit aggregate.
 us,scenario_095,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -5686,101 +5971,106 @@ us,scenario_099,child2_wic_eligible,claude-haiku-4.5,llm_error,categorical_eligi
 us,scenario_099,child2_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the under-five age requirement as the complete WIC eligibility test. Child 2 satisfies the categorical age rule, but household income of approximately $159,676 exceeds the 185%-of-poverty income limit, making the child ineligible."
 us,scenario_099,child2_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model inferred eligibility solely from child 2's age and never applied WIC's household-income test. The household's approximately $159,676 annual income is above the applicable 185%-of-poverty limit, so the categorical age condition does not produce eligibility."
 us,scenario_099,child2_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for child2_wic_eligible, violating the required submission contract. It therefore never reported the result produced by applying the WIC income limit: value 0."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model used a $32,600 standard deduction instead of $34,200 and assigned the CDCC a 20% rate instead of the applicable 32% rate. It also introduced unsupported non-itemizer charitable and auto-loan-interest deductions, preventing the required $125,385.05 taxable-income calculation."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model failed to reduce wages for the pre-tax 401(k) contributions, omitted the net capital-loss and above-the-line deductions, and used a $29,200 standard deduction rather than $34,200. It also reduced tax by only $3,174 instead of the full $6,320 of nonrefundable CTC and CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model substituted estimated brackets and a $31,500 standard deduction for the applicable 2026 parameters, then used a $4,000 CTC and 20% CDCC. The correct credits are $4,400 and $1,920, and the qualified dividends generate $19.20 of tax rather than zero."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model's own intermediate calculation reached about $12,421 after credits, but it submitted $18,962, a number incompatible with that calculation. It also double-counted the $128 qualified dividend in gross income and used outdated deduction and credit parameters."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model omitted the CDCC entirely and subtracted only a $4,000 CTC. The household receives a $1,920 CDCC at the 32% rate and a $4,400 CTC, reducing $16,999.75 of pre-credit tax to $10,679.75."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly treated the refundable limit of the CTC as splitting an otherwise usable credit into only $600 nonrefundable and $3,400 refundable. Because pre-credit tax is sufficient, the full $4,400 CTC is used nonrefundably here; the model also used a 20% CDCC rate instead of 32%."
-us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly described credits that reduce its estimated pre-credit tax, then submitted $24,500, which exceeds that pre-credit tax and contradicts its own computation. It further invoked AMT, NIIT, and itemized-deduction add-backs that do not belong in this output."
-us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced AGI to $134,987, claimed personal exemptions, and itemized deductions instead of applying the $34,200 standard deduction. It also used only $750 of CTC and a 20% CDCC rather than $4,400 and $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied a TCJA-expiration regime with itemized deductions and personal exemptions instead of the applicable $34,200 standard deduction. It also used a $2,000 CTC and 20% CDCC, understating the nonrefundable credits by $3,120."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $164,481 AGI omits the net capital-loss deduction and other above-the-line adjustments that produce $159,585.05. Its submitted amount also leaves the $6,320 of nonrefundable CTC and CDCC unapplied."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an inapplicable post-sunset regime with personal exemptions and itemized deductions, producing an erroneous $135,169 AGI and $82,947 taxable income. It then reduced the CTC to $700 and used a 20% CDCC rather than the applicable $4,400 and $1,920 credits."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used personal exemptions, itemized deductions, and pre-TCJA tax treatment, reducing taxable income to $84,650 instead of $125,385.05. It also limited the credits to a $750 CTC and $1,200 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The answer supplies no derivation for its $11,105 figure. The required chain is $16,999.75 of tax before credits minus $4,400 of CTC and $1,920 of CDCC, which yields $10,679.75."
-us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model expressly applied personal exemptions and itemized deductions, neither of which determines this household's taxable income under the applicable rules. Taxable income instead equals $159,585.05 of AGI minus the $34,200 standard deduction."
-us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model invented a $15,600 CTC and an income-ineligible $1,404 saver's credit, while treating only one child as qualifying for the childcare expense cap. Both children support a $6,000 CDCC expense base, and the actual nonrefundable credits total $6,320."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The submitted $17,278 exceeds the $16,999.75 tax before credits, so the model did not apply the household's usable nonrefundable credits correctly. The two qualifying children and childcare expenses generate $4,400 of CTC and $1,920 of CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The answer gives no specific credit calculation and leaves $14,696 after vaguely invoking child-related credits. The required nonrefundable reduction is exactly $6,320: $4,400 of CTC plus $1,920 of CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model overstated AGI, used a $32,200 standard deduction instead of $34,200, and therefore overstated taxable income and pre-credit tax. It also calculated the CDCC at 20% rather than 32%, subtracting $1,200 instead of $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Although the model identified the correct $4,400 CTC, it used only a $1,200 CDCC instead of $1,920 and did not apply the exact $34,200 standard deduction and 2026 brackets. Those parameters produce $16,999.75 before credits and $10,679.75 afterward."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used taxable income of $127,528 instead of $125,385.05 and consequently overstated regular tax. It also applied a 20% CDCC of $1,200 instead of the applicable 32% credit of $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model used an approximate $128,000 taxable income instead of $125,385.05 and claimed itemized deductions despite the $34,200 standard deduction being controlling. It also used a $4,000 CTC and $1,200 CDCC instead of $4,400 and $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model replaced the required calculation with a broad family-income estimate. Exact application of the $34,200 standard deduction, 2026 rates, $4,400 CTC, and $1,920 CDCC yields $10,679.75."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied restored 2017-style brackets, personal exemptions, and a fully phased-out CTC. Under the applicable 2026 rules there are no personal exemptions in this calculation, and the household receives the full $4,400 CTC plus a $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used obsolete 10%, 15%, and 25% brackets and an erroneous $134,987 AGI, then phaseout-reduced the CTC to $751. The applicable computation uses $159,585.05 of AGI, current brackets, a $4,400 CTC, and a $1,920 CDCC."
-us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used a $32,200 standard deduction rather than $34,200, overstating taxable income and pre-credit tax. It also applied a 20% CDCC of $1,200 rather than the 32% credit of $1,920."
-us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
-us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
-us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted that charitable and other deductions eliminate the liability, but the household takes the $34,200 standard deduction and retains $125,385.05 of taxable income. After the full $6,320 of nonrefundable credits, $10,679.75 remains."
-us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's own revised computation produced approximately $11,730 after credits, but it submitted $24,711.50, contradicting its calculation. It also deducted the full $6,478 capital loss instead of the $3,000 annual limit and used outdated standard-deduction and credit amounts."
-us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI to $164,894 by miscomputing both 401(k) contributions and above-the-line deductions, then used a $29,994 standard deduction rather than $34,200. It also omitted the $4,400 CTC and treated the CDCC as $1,200 rather than $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model applied a 20% CDCC rate and subtracted only $1,200 instead of the trace's 32% rate and $1,920 credit. It also used estimated deductions and brackets, then submitted $12,103 despite its own calculation ending near $10,965."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model failed to reduce wages for the pre-tax 401(k) contributions, recognize the net capital-loss treatment, and apply the $34,200 standard deduction, overstating taxable income as $136,302. It also invented a $3,174 credit total instead of subtracting the $4,400 CTC and $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used an estimated $31,500 standard deduction and estimated tax brackets instead of the $34,200 deduction and applicable 2026 parameters. It also used a $4,000 CTC and 20% CDCC, understating nonrefundable credits by $1,120."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The submitted $18,962 contradicts the model's own derived after-credit result of approximately $12,421. Its derivation also used a $31,500 standard deduction, a $4,000 CTC, and a 20% CDCC rather than $34,200, $4,400, and 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model omitted the CDCC even though two qualifying children and $12,740 of childcare expenses produce a $1,920 nonrefundable credit. It also used approximate taxable income and tax parameters rather than the traced $125,385.05 and $16,999.75."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model incorrectly treated the potentially refundable ceiling of the CTC as automatically dividing the credit into only $600 nonrefundable and $3,400 refundable. With sufficient tax liability, the full $4,400 CTC is used nonrefundably, and the CDCC is $1,920 at 32%, not $1,200 at 20%."
+us,scenario_099,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The submitted $24,500 is incompatible with the model's own approximately $19,900 tax-before-credit calculation and its stated child-related credits. It additionally invoked AMT, NIIT, and itemized-deduction effects that do not enter this output and failed to apply the traced $6,320 of nonrefundable credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly introduced personal exemptions and itemized deductions, producing $88,929 of taxable income instead of $125,385.05 under the $34,200 standard deduction. It also reduced the CTC to $750 and the CDCC to $1,200 instead of applying $4,400 and $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model applied a post-TCJA-expiration regime with personal exemptions and itemized deductions rather than the traced $34,200 standard deduction. It also used a $2,000 CTC and 20% CDCC instead of the $4,400 CTC and 32% CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $164,481 AGI omits material reductions reflected in the trace, including the net capital loss and above-the-line deductions, since the correct AGI is $159,585.05. The answer also fails to reflect subtraction of the full $6,320 in nonrefundable credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used sunset-era personal exemptions and itemized deductions to reduce taxable income to $82,947 rather than using the $34,200 standard deduction. It then phaseout-reduced the CTC to $700 and used a $1,200 CDCC instead of $4,400 and $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model applied personal exemptions and itemized deductions, driving taxable income down to $84,650 instead of the traced $125,385.05. It also used a phaseout-reduced $750 CTC and $1,200 CDCC rather than the full $4,400 and $1,920 credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The unexplained $11,105 does not follow the traced computation of $16,999.75 less $6,320. The answer embodies an incorrect tax-before-credit amount, credit total, or both."
+us,scenario_099,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model expressly applied personal exemptions and itemized deductions, neither of which produces the traced taxable income. The required computation uses the $34,200 standard deduction and then subtracts $6,320 of nonrefundable credits."
+us,scenario_099,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model invented a $15,600 CTC and a $1,404 saver's credit despite the applicable credits being a $4,400 CTC and $1,920 CDCC. It also treated only one child as qualifying for the childcare expense cap and mishandled the capital loss and pre-tax deductions."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The $17,278 answer is close to tax before credits and therefore fails to subtract the traced $6,320 of nonrefundable credits. Those credits are the $4,400 CTC and $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The answer's generic reference to child-related credits does not implement the actual $6,320 reduction. The correct credit step subtracts $4,400 of CTC and $1,920 of CDCC from $16,999.75."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a $32,200 standard deduction rather than $34,200 and consequently overstated taxable income and regular tax. It also applied a 20% CDCC of $1,200 instead of the 32% credit of $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"Although the model identified the $4,400 CTC, it applied only a $1,200 CDCC rather than $1,920 at the applicable 32% rate. Its remaining excess also reflects an overstated tax base or tax from failing to reproduce the $125,385.05 taxable income and $16,999.75 pre-credit tax."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model overstated taxable income as $127,528 instead of $125,385.05 and tax before credits as $17,471.20 instead of $16,999.75. It then used a 20% CDCC of $1,200 rather than the applicable $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model used a $4,000 CTC and $1,200 CDCC, understating the traced nonrefundable credits by $1,120. It also described itemized deductions despite the computation using the $34,200 standard deduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The rough family-of-four shortcut did not calculate the traced $125,385.05 taxable income or the $6,320 credit reduction. Applying those steps to the $16,999.75 pre-credit tax yields $10,679.75, not $14,000."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied restored 2017-style deductions, exemptions, brackets, and CTC phaseout rules. The applicable computation uses a $34,200 standard deduction, current ordinary rates, a full $4,400 CTC, and a $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-sunset regime with personal exemptions, itemization, 10/15/25% brackets, and a fully phased-out CTC. The traced regime instead produces $125,385.05 of taxable income and allows the full $4,400 CTC plus a $1,920 CDCC."
+us,scenario_099,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model phaseout-reduced the CTC to $751 and applied a 20% CDCC of $1,200. The household receives a $4,400 nonrefundable CTC and a $1,920 CDCC, with no such CTC reduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used a $32,200 standard deduction instead of $34,200 and therefore overstated taxable income and tax before credits. It also used a 20% CDCC of $1,200 rather than the applicable 32% credit of $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or reasoning for the requested variable.
+us,scenario_099,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no numeric output or reasoning for the requested variable.
+us,scenario_099,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated charitable and other deductions and nonrefundable credits as eliminating the liability. The standard-deduction computation leaves $125,385.05 taxable income and $10,679.75 after the $6,320 credit reduction."
+us,scenario_099,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used a $32,200 standard deduction and estimated brackets, overstating taxable income and tax before credits. It also applied a 20% CDCC of $1,200 rather than $1,920 at 32%."
+us,scenario_099,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The submitted $24,711.50 contradicts the model's own final derivation of approximately $11,730. That derivation also deducted the full $6,478 capital loss directly, used estimated deduction and bracket parameters, and allowed only a $4,000 CTC and $1,200 CDCC instead of $4,400 and $1,920."
+us,scenario_099,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as $164,894 by miscomputing the pre-tax 401(k), educator, capital-loss, and other above-the-line adjustments, and it used a $29,994 standard deduction instead of $34,200. It also failed to subtract the $4,400 CTC and misstated the CDCC as $1,200."
 us,scenario_099,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated the ACTC earned-income formula as creating a refundable payment even though the household's tax liability fully absorbs the CTC as a nonrefundable credit. Its asserted $1,200 is unsupported by its own calculation: no unused CTC remains to become refundable, so refundable CTC is $0."
 us,scenario_099,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, violating the required submit_outputs contract and leaving the requested value missing."
 us,scenario_099,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no output for federal_refundable_credits, violating the required submit_outputs contract and leaving the requested value missing."
 us,scenario_099,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model correctly stated that the dependent credits were nonrefundable and fully absorbed by tax liability, then incorrectly reported their $600 amount as federal_refundable_credits. A nonrefundable credit cannot enter this output, and its own reasoning therefore yields $0 rather than $600."
-us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,The model applied the federal 130% FPL income test as dispositive and omitted California's universal free school meals program. That state program supersedes the income test for the enrolled K-12 child.
-us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly made the federal 130% FPL test—and even household assets—determinative. California provides universal free school meals to enrolled students without an income or asset test.
-us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,The model treated the federal income thresholds as controlling and omitted California's universal free school meals program. The enrolled K-12 child qualifies regardless of the household's income.
-us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,"The model stopped after comparing household income with the federal 130% FPL threshold. It failed to apply California's universal program, which produces free-meal support regardless of income."
-us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,The model treated failure of both income-based and categorical eligibility as conclusive. California's universal free school meals pathway independently qualifies the enrolled student.
-us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,The model applied only the National School Lunch Program's 130% FPL rule and omitted California's universal free school meals rule. The state rule qualifies every enrolled K-12 student regardless of income.
-us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,The model relied on the federal 130% and 185% FPL thresholds and failed to apply California's universal free school meals program. The enrolled nine-year-old receives free-meal support despite the household's high income.
-us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,The model treated income above 130% FPL as disqualifying. California's universal program overrides that income-based result for enrolled K-12 students.
-us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The model used an income threshold as the sole eligibility route. It omitted California's universal free school meals program, under which the enrolled child qualifies without an income test."
-us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model incorrectly treated exceeding 130% FPL as dispositive. California universally provides free school meals to enrolled students, so the household's 5.03 FPG ratio does not bar support."
-us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,The model applied the ordinary income limit and omitted California's universal free school meals rule. The enrolled K-12 child qualifies regardless of household income.
-us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,The model stopped at the federal 130% FPL comparison. California's universal program supersedes that test and supplies free meals to the enrolled student.
-us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model made its adjusted-income estimate and the 130% FPL threshold controlling. It failed to apply California's universal free school meals program, which does not condition the enrolled child's meals on income."
-us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,The model inferred no benefit solely from income-based eligibility thresholds. California's universal free school meals program creates positive support for the enrolled child regardless of income.
-us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,The model treated the absence of an explicit eligibility fact and the household's income as disqualifying. The stated California residence activates the state's universal free school meals rule for the enrolled K-12 child.
-us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model assumed that failing the 130% FPL test and categorical eligibility exhausted all pathways. It omitted California's universal free school meals pathway, which independently produces positive support."
-us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model treated the ordinary income threshold as dispositive. California's universal program provides free meals to the enrolled K-12 child without regard to that threshold.
-us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,"The model denied eligibility based only on high income. It omitted California's universal free school meals program, which qualifies enrolled students at every income level."
-us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,The model treated high household income as an automatic bar. California's universal free school meals program makes income irrelevant for the enrolled K-12 student.
-us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model treated failure of the 130% FPL and categorical-eligibility tests as conclusive. California's universal program is a separate qualifying pathway and generates positive free-meal support.
-us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,The model applied the federal 130% FPL ceiling without applying California's universal free school meals rule. The enrolled child qualifies despite household income above that ceiling.
-us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,"The model considered only income eligibility and categorical routes such as SNAP or TANF. It omitted California's universal free school meals pathway, which requires neither for an enrolled student."
-us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied neither the required numeric value nor an explanation for free_school_meals_eligible. Its response therefore failed the output contract before substantive eligibility reasoning could be evaluated.
-us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,The model treated the federal income limits as controlling. California's universal free school meals program qualifies the enrolled K-12 child regardless of household income.
-us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,"The model relied exclusively on household income exceeding 130% FPL. It failed to apply California's universal free school meals rule, which supersedes that income test for enrolled students."
-us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,The model denied support solely because income exceeded the ordinary threshold. California's universal program provides positive free-meal support to the enrolled child regardless of income.
+us,scenario_099,free_school_meals_eligible,claude-fable-5,llm_error,state_local_rule,False,The model applied the federal 130% FPL income test as dispositive and omitted California's universal free school meals program. That state program supersedes the income result for the enrolled K–12 child.
+us,scenario_099,free_school_meals_eligible,claude-haiku-4.5,llm_error,state_local_rule,False,The model incorrectly conditioned California free meals on the federal 130% FPL test and cited assets that do not restrict the universal program. California provides free school meals to the enrolled child regardless of this household's income or assets.
+us,scenario_099,free_school_meals_eligible,claude-opus-4.7,llm_error,state_local_rule,False,The model treated an income threshold—incorrectly identifying 185% FPL as the free-meal cutoff—as controlling. California's universal program makes the enrolled student eligible regardless of either the 130% or 185% threshold.
+us,scenario_099,free_school_meals_eligible,claude-opus-4.8,llm_error,state_local_rule,False,The model stopped after applying the ordinary 130% FPL test and omitted California's universal free-meals rule. The enrolled K–12 child receives positive free-meal support despite the household's high income.
+us,scenario_099,free_school_meals_eligible,claude-opus-5,llm_error,state_local_rule,False,"The model treated failure of both income and categorical eligibility tests as conclusive. It omitted the independent California universal-meals pathway, which qualifies the enrolled child."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied the ordinary National School Lunch Program 130% FPL limit without applying California's universal program. Under the state rule, household income does not bar the enrolled student from free meals."
+us,scenario_099,free_school_meals_eligible,claude-sonnet-5,llm_error,state_local_rule,False,The model relied exclusively on the federal 130% and 185% FPL thresholds. California's universal free school meals program overrides those income-screening outcomes for the enrolled child.
+us,scenario_099,free_school_meals_eligible,deepseek-v4-pro,llm_error,state_local_rule,False,The model treated income above 130% FPL as disqualifying and omitted California's universal eligibility rule. The enrolled student qualifies without an income test.
+us,scenario_099,free_school_meals_eligible,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model inferred ineligibility solely from income exceeding the ordinary threshold. It failed to apply California's universal free school meals program to the enrolled child.
+us,scenario_099,free_school_meals_eligible,gemini-3.1-pro-preview,llm_error,state_local_rule,False,The model made the 130% FPL threshold dispositive. California's universal program supplies a separate eligibility pathway regardless of the household's 5.03 FPG ratio.
+us,scenario_099,free_school_meals_eligible,gemini-3.6-flash,llm_error,state_local_rule,False,The model used the income threshold while omitting California's universal free-meals rule. That state rule makes the enrolled K–12 child eligible.
+us,scenario_099,free_school_meals_eligible,gemini-3.7-flash,llm_error,state_local_rule,False,"The model treated exceeding 130% of the federal poverty guideline as a bar. California universally provides free school meals to enrolled students, so this income comparison does not control."
+us,scenario_099,free_school_meals_eligible,glm-5.2,llm_error,state_local_rule,False,"The model focused on its adjusted-gross-income estimate and the 130% FPL test, neither of which governs California's universal program. The enrolled child qualifies regardless of that calculated income."
+us,scenario_099,free_school_meals_eligible,gpt-5.4-mini,llm_error,state_local_rule,False,The model assumed the conventional income limits controlled and omitted California's universal free school meals program. The state program produces positive support for the enrolled child at this income.
+us,scenario_099,free_school_meals_eligible,gpt-5.4-nano,llm_error,state_local_rule,False,The model wrongly required the prompt to state conventional eligibility facts and relied on high wages to return zero. The listed California residence activates universal free-school-meal eligibility for the enrolled student.
+us,scenario_099,free_school_meals_eligible,gpt-5.5,llm_error,state_local_rule,False,"The model treated failure of the 130% FPL and categorical-eligibility routes as exhaustive. It omitted California's universal program, an independent route that yields positive free-meal support."
+us,scenario_099,free_school_meals_eligible,gpt-5.6-luna,llm_error,state_local_rule,False,The model made the federal income threshold controlling and failed to apply California's universal state program. The enrolled child qualifies even though household income exceeds the threshold.
+us,scenario_099,free_school_meals_eligible,gpt-5.6-terra,llm_error,state_local_rule,False,The model denied eligibility based only on high income. California's universal free school meals program removes the income condition for the enrolled child.
+us,scenario_099,free_school_meals_eligible,grok-4.3,llm_error,state_local_rule,False,"The model treated high household income as disqualifying. It omitted California's universal program, under which the enrolled K–12 student qualifies regardless of income."
+us,scenario_099,free_school_meals_eligible,grok-4.5,llm_error,state_local_rule,False,The model treated the 130% FPL and categorical-eligibility pathways as exhaustive and consequently predicted zero support. California's universal-meals pathway independently qualifies the enrolled child.
+us,scenario_099,free_school_meals_eligible,grok-build-0.1,llm_error,state_local_rule,False,"The model applied the federal 130% FPL cutoff without the California universal-program override. The enrolled child remains eligible despite household income near $160,000."
+us,scenario_099,free_school_meals_eligible,kimi-k2.6,llm_error,state_local_rule,False,The model correctly identified failure of the ordinary income and categorical tests but incorrectly treated those routes as exhaustive. California's universal free school meals program is the independent pathway that produces positive support.
+us,scenario_099,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,The model supplied neither the required numeric value nor an explanation for free_school_meals_eligible. It therefore failed the output contract before any substantive eligibility determination could be evaluated.
+us,scenario_099,free_school_meals_eligible,minimax-m3,llm_error,state_local_rule,False,"The model used income limits as the sole eligibility rule. It omitted California's universal free school meals program, which covers the enrolled student regardless of household income."
+us,scenario_099,free_school_meals_eligible,ox-alpha,llm_error,state_local_rule,False,The model treated the 130% FPL cutoff as a necessary condition for positive support. California's universal program eliminates that income condition for the enrolled child.
+us,scenario_099,free_school_meals_eligible,qwen-3.7-max,llm_error,state_local_rule,False,The model applied the federal 130% FPL threshold as dispositive and omitted California's universal state rule. The enrolled K–12 child qualifies even at the household's stated wage and investment income.
+us,scenario_099,free_school_meals_eligible,qwen3.8-max,llm_error,state_local_rule,False,The model inferred ineligibility from income above the ordinary threshold. California's universal free school meals program instead makes the enrolled child eligible without regard to income.
 us,scenario_099,head_chip_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,head_medicaid_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_099,payroll_tax,claude-fable-5,llm_error,payroll_tax_base,False,"The model used a 1.2% California SDI rate instead of the applicable 1.3%, understating SDI by $165. It then submitted $14,118 even though its own components summed to $14,602.50, adding a separate arithmetic-to-output inconsistency."
-us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated California as having no mandatory employee payroll tax and omitted the $2,145 California SDI contribution. Federal FICA of $12,622.50 plus SDI produces $14,767.50."
-us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model used an outdated 1.2% California SDI rate instead of 1.3%, calculating $1,980 rather than $2,145. It also submitted $13,197.50 despite explicitly recomputing its own components as $14,602.50."
-us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,payroll_tax_base,False,The model calculated California SDI using 1.1% rather than 1.3% and then excluded SDI altogether on the unsupported ground that the output focused on federal FICA. The requested payroll-tax definition expressly includes mandatory employee state payroll taxes.
-us,scenario_099,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model omitted the mandatory $2,145 California SDI contribution and rounded Medicare down from $2,392.50 to $2,392. Federal FICA alone is $12,622.50, and including SDI yields $14,767.50."
-us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,The model correctly used gross wages for FICA but applied a 1.1% California SDI rate instead of 1.3%. This understated the state contribution by $330 and the total by the same amount.
-us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model vacillated between omitting California SDI and applying an outdated 1.1% rate, then submitted $12,903.50, which follows neither calculation. California SDI is mandatory at 1.3% of $165,000, adding $2,145 to federal FICA."
-us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted $24,597 of listed employer-sponsored insurance premiums from wages even though the benchmark uses the stated $165,000 of gross wages as the payroll-tax base. It also used a 1.1% California SDI rate instead of 1.3%."
-us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,payroll_tax_base,False,"The model applied a 1.1% California SDI rate instead of the applicable 1.3%. SDI is $2,145 rather than $1,815, increasing the total from $14,437.50 to $14,767.50."
-us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"The model computed only federal Social Security and Medicare taxes and omitted the mandatory $2,145 California SDI contribution. The output definition includes mandatory employee state payroll taxes."
-us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced the $165,000 payroll-tax base by $24,597 of listed employer-sponsored insurance premiums. It also applied a 1.1% California SDI rate rather than 1.3%; the trace taxes the full gross wages and produces $2,145 of SDI."
-us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model applied a 1.1% California SDI rate rather than 1.3%, producing $1,815 instead of $2,145. Its federal FICA components were otherwise correct."
-us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The submitted $13,280 does not equal the required $12,622.50 of federal FICA plus $2,145 of California SDI. The answer implies that only $657.50 of state payroll tax was added, rather than applying the 1.3% SDI rate to all $165,000 of wages."
-us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Although the model named Social Security, Medicare, and California SDI, its $12,425.67 total is even below federal FICA alone. It failed to apply the stated gross-wage base and the 1.3% California SDI computation that yield $14,767.50."
-us,scenario_099,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly treated the Social Security wage base as a household-wide cap even though the cap applies separately to each worker, and neither worker individually reaches it. It also omitted the mandatory $2,145 California SDI contribution and submitted a value inconsistent with all of its intermediate totals."
-us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model omitted the mandatory $2,145 California SDI contribution and rounded federal Medicare tax down by $0.50. The requested output includes state employee payroll taxes, making the total $14,767.50."
-us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model stated that it calculated only employee Social Security and Medicare but reported $13,577 rather than the correct federal FICA amount of $12,622.50. It also omitted the separately required $2,145 California SDI contribution."
-us,scenario_099,payroll_tax,gpt-5.5,llm_error,payroll_tax_base,False,"The model applied a 1.2% California SDI rate instead of 1.3%. This produced $1,980 of SDI rather than $2,145 and understated payroll tax by $165."
-us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model used an approximate, incorrect California SDI computation rather than applying 1.3% to the full $165,000 of wages. Federal FICA is $12,622.50 and exact SDI is $2,145, not the $1,689.90 embedded in its total."
-us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,payroll_tax_base,False,"The model used a 1.2% California SDI rate instead of 1.3%, calculating $1,980 rather than $2,145. Its federal FICA calculation was correct, leaving a $165 understatement."
-us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model used an approximate 1.2% California SDI amount of $1,980 instead of the required 1.3% amount of $2,145. Rounding Medicare to $2,393 does not cure the resulting $164.50 understatement."
-us,scenario_099,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"The model approximated federal FICA as $12,623 and omitted the mandatory $2,145 California SDI contribution. Exact federal FICA is $12,622.50, and the state component brings the total to $14,767.50."
-us,scenario_099,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model applied a 1.1% California SDI rate instead of 1.3%, understating SDI by $330. It also rounded the resulting outdated-rate total from $14,437.50 to $14,438."
-us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted $24,597 of listed employer-sponsored insurance premiums from the gross-wage payroll-tax base. It then omitted California SDI entirely; the correct computation taxes $165,000 and adds $2,145 of mandatory state payroll tax."
-us,scenario_099,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model used a 1.2% California SDI rate instead of 1.3%, producing $1,980 rather than $2,145. It then rounded its $14,602.50 intermediate result to $14,603, leaving the underlying $165 rate error intact."
-us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing."
-us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing."
-us,scenario_099,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model correctly computed federal FICA components of $6,502.50 and $6,120 but then submitted $25,290, exactly double their $12,622.50 sum. It also omitted the $2,145 California SDI contribution."
-us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model recognized that mandatory California SDI belongs in the output but ultimately submitted federal FICA alone. Its alternative SDI calculation also used 1.1% instead of 1.3%; the required state contribution is $2,145."
-us,scenario_099,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model applied California SDI only to the head and calculated even that component as $472.20, which is not 1.2% of $85,000. SDI applies at 1.3% to both workers' $165,000 of gross wages, producing $2,145."
+us,scenario_099,payroll_tax,claude-fable-5,llm_error,state_local_rule,False,"The model used a 1.2% California SDI rate instead of the applicable rate yielding $2,145, then submitted $14,118 despite its own calculation yielding $14,602.50. It therefore both misapplied the state payroll-tax rate and broke its stated arithmetic."
+us,scenario_099,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly treated California as having no mandatory employee payroll tax and omitted the $2,145 employee SDI contribution. Its $12,622.50 answer contains only Social Security and Medicare."
+us,scenario_099,payroll_tax,claude-opus-4.7,llm_error,state_local_rule,False,"The model used a 1.2% California SDI rate rather than the rate producing $2,145 and then submitted $13,197.50 even though its displayed components sum to $14,602.50. The final value is inconsistent with its own computation."
+us,scenario_099,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model explicitly excluded California SDI after recognizing that it belongs in the calculation. It also used 1.1% rather than the applicable rate yielding $2,145."
+us,scenario_099,payroll_tax,claude-opus-5,llm_error,state_local_rule,False,"The model calculated only federal Social Security and Medicare, rounded Medicare down by $0.50, and omitted the $2,145 California SDI contribution."
+us,scenario_099,payroll_tax,claude-sonnet-4.6,llm_error,state_local_rule,False,"The federal FICA calculation is correct, but the model applied a 1.1% California SDI rate and obtained $1,815. The applicable California contribution is $2,145, leaving its answer $330 too low."
+us,scenario_099,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model vacillated between omitting California SDI and applying an incorrect 1.1% rate, then submitted $12,903.50, which follows neither computation. California SDI is mandatory here and equals $2,145."
+us,scenario_099,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model deducted $24,597 of employer-sponsored insurance premiums from the stated gross wages when establishing the Social Security, Medicare, and SDI bases. The trace applies all three taxes to the full $165,000 of wages, including $2,145 of California SDI."
+us,scenario_099,payroll_tax,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly computed federal FICA but applied a 1.1% California SDI rate, producing $1,815 instead of $2,145. This understated payroll tax by $330."
+us,scenario_099,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The answer includes only the $10,230 Social Security tax and $2,392.50 Medicare tax. It omits the mandatory $2,145 California SDI contribution."
+us,scenario_099,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model improperly reduced payroll-tax wages by the listed employer-sponsored insurance premiums, using $140,403 rather than $165,000. It also applied a 1.1% California SDI rate instead of the contribution yielding $2,145."
+us,scenario_099,payroll_tax,gemini-3.5-flash,llm_error,state_local_rule,False,"The model used 1.1% for California SDI and calculated $1,815. The applicable state contribution is $2,145, so the otherwise correct federal FICA total was understated by $330."
+us,scenario_099,payroll_tax,gemini-3.6-flash,llm_error,state_local_rule,False,"The stated $13,280 total leaves only $657.50 above the correctly derived $12,622.50 of federal FICA, so it fails to include the full $2,145 California SDI contribution. The correct three-component total is $14,767.50."
+us,scenario_099,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model named all three tax components but submitted less than the $12,622.50 federal Social Security and Medicare liability alone. It therefore used an unjustified reduced wage base or otherwise undercomputed FICA before also failing to add the full $2,145 California SDI amount."
+us,scenario_099,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly treated the Social Security wage base as a household-wide cap, although the cap applies separately to each worker and both individual wages are below it. It also omitted California SDI and submitted a value unsupported by any of its displayed calculations."
+us,scenario_099,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model included only Social Security and Medicare and omitted the mandatory $2,145 California SDI contribution. It also rounded the exact $12,622.50 federal FICA subtotal down to $12,622."
+us,scenario_099,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model claimed to sum ordinary employee FICA but $165,000 at 7.65% equals $12,622.50, not $13,577. It also failed to separately compute and add the $2,145 California SDI contribution."
+us,scenario_099,payroll_tax,gpt-5.5,llm_error,state_local_rule,False,"The model correctly calculated $12,622.50 of federal FICA but used a 1.2% California SDI rate, producing $1,980. California SDI is $2,145, so the answer is $165 too low."
+us,scenario_099,payroll_tax,gpt-5.6-luna,llm_error,state_local_rule,False,"The model estimated California SDI rather than applying the required state contribution of $2,145. Its total implies only $1,689.90 of SDI above the correct $12,622.50 federal FICA subtotal."
+us,scenario_099,payroll_tax,gpt-5.6-sol,llm_error,state_local_rule,False,"The model used $1,980 of California SDI based on a 1.2% rate. The applicable contribution is $2,145, making its total $165 too low."
+us,scenario_099,payroll_tax,gpt-5.6-terra,llm_error,state_local_rule,False,"The model used approximately $1,980 of California SDI rather than $2,145 and rounded Medicare to $2,393. The state-rate error leaves the submitted total $164.50 below the required amount."
+us,scenario_099,payroll_tax,grok-4.3,llm_error,state_local_rule,False,"The model approximated federal FICA at $12,623 and omitted the mandatory $2,145 California SDI contribution. Exact Social Security and Medicare total $12,622.50 before SDI."
+us,scenario_099,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model applied a 1.1% California SDI rate, yielding $1,815 instead of $2,145. It then rounded the resulting $14,437.50 subtotal to $14,438."
+us,scenario_099,payroll_tax,grok-4.6,llm_error,state_local_rule,False,"The model correctly computed $12,622.50 of federal FICA but used a 1.2% California SDI rate and obtained $1,980. The applicable California contribution is $2,145."
+us,scenario_099,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model improperly subtracted employer-sponsored insurance premiums from gross wages and computed FICA on $140,403 rather than $165,000. It also omitted the mandatory $2,145 California SDI contribution."
+us,scenario_099,payroll_tax,inkling,llm_error,state_local_rule,False,"The model used 1.2% for California SDI, calculating $1,980 rather than $2,145. It also rounded the resulting $14,602.50 to $14,603."
+us,scenario_099,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so the required structured output was missing."
+us,scenario_099,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no payroll_tax value or explanation, so the required structured output was missing."
+us,scenario_099,payroll_tax,minimax-m3,llm_error,other,False,"The model's own per-worker FICA figures sum to $12,622.50, not $25,290. It nearly doubled the federal total and also failed to add the distinct $2,145 California SDI contribution correctly."
+us,scenario_099,payroll_tax,ox-alpha,llm_error,state_local_rule,False,"The model incorrectly asserted that no employee-side state payroll tax applies. California SDI is mandatory for this output and adds $2,145 to the correctly calculated $12,622.50 of federal FICA."
+us,scenario_099,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model recognized that mandatory California SDI must be included but discarded it in the submitted value. It also used a 1.1% SDI rate rather than the applicable contribution of $2,145."
+us,scenario_099,payroll_tax,qwen3.8-max,llm_error,state_local_rule,False,"The model applied California SDI only to the head and calculated even that amount incorrectly: 1.2% of $85,000 is $1,020, not $472.20. SDI applies across both workers' $165,000 of wages and totals $2,145."
 us,scenario_099,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -5790,36 +6080,38 @@ us,scenario_099,spouse_medicaid_eligible,kimi-k3,parse_contract_failure,missing_
 us,scenario_099,spouse_medicare_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,spouse_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income close to the traced amount but overstated bracket tax, claiming roughly $6,940 instead of $5,778.21. It then inconsistently submitted $6,222 despite its own stated post-credit estimate of $5,684."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $4,754 married-filing-jointly standard deduction instead of the larger $20,476.57 California itemized deduction. It also treated unreimbursed employee expenses as an above-the-line subtraction and failed to apply the $1,284.46 dependent exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached taxable income near the traced $139,108.47 but never completed the California bracket computation consistently, cycling through incompatible base-tax formulas. Its submitted $5,959 also contradicts its own stated post-credit estimate of $4,680."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to subtract the $3,088 of traditional 401(k) deferrals from the listed gross wages, inflating California AGI and taxable income. It then overstated bracket tax at roughly $7,973 rather than $5,778.21."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model applied California's married-joint brackets incorrectly, assigning roughly $7,700 of tax to about $138,000 of taxable income instead of $5,778.21 on $139,108.47. It also misstated the exemption-credit arithmetic before submitting $7,250."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model inflated the 2026 bracket thresholds by an unsupported 4% and used exemption credits of roughly $1,800 rather than $1,284.46. Those two parameter errors reduced its result below $4,493.74 despite deriving taxable income close to the trace."
-us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income near $150,000 instead of deriving $139,108.47 from $159,585.05 of AGI and $20,476.57 of itemized deductions. It also failed to subtract the full $1,284.46 of nonrefundable exemption credits from the correctly bracketed tax."
-us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $112,436 instead of $139,108.47, implying more than $47,000 of reductions rather than the traced $20,476.57 itemized deduction. That unsupported income reduction drove both its $4,551.40 pre-credit tax and $3,263 result too low."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model subtracted only about $1,180 of California exemption credits instead of $1,284.46. It also failed to reproduce the traced $5,778.21 bracket tax, yielding $4,665 rather than $4,493.74."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $7,845 exceeds even the traced $5,778.21 tax before credits, so the model did not apply the 2026 California married-joint brackets to $139,108.47 of taxable income. It also omitted the subtraction of $1,284.46 in nonrefundable exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated California AGI by more than $24,000 and overstated itemized deductions at $26,138, producing taxable income of $109,031 instead of $139,108.47. It then subtracted only about $600 of exemption credits rather than $1,284.46."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The $2,570 result is inconsistent with the traced $5,778.21 bracket tax less $1,284.46 of exemption credits. The model's generic explanation omitted the actual taxable-income, bracket, and credit computations and materially understated the liability."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model submitted $4,128 without performing the required calculation from $139,108.47 of taxable income. Correct bracket application yields $5,778.21 before subtracting $1,284.46 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model's $4,056 answer understates the result of applying the 2026 married-joint brackets and exemption credits. The required sequence is $5,778.21 of bracket tax minus $1,284.46, not an unsupported aggregate estimate."
-us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model wrongly used a $6,566 standard deduction and claimed charitable contributions were deductible only to the extent exceeding that amount; California instead allows the full applicable itemized deductions, totaling $20,476.57. It also treated the $338 educator expense as a tax credit, and its submitted $3,202.96 contradicts its own $5,559 calculation."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model's $6,713 exceeds the traced $5,778.21 tax before any credits, showing that it overstated California bracket tax. It also failed to subtract the $1,284.46 of nonrefundable exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model's unsupported $4,931 result is $437.26 above the amount obtained from the traced bracket and credit computation. It did not reproduce $5,778.21 of pre-credit tax less $1,284.46 of exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly reduced AGI using health-premium exclusions and other deductions, then reported internally inconsistent taxable income figures. Despite claiming taxable income near $101,811, it produced $6,882 of tax, which also reflects a severe bracket computation error."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model said it used the California standard deduction even though the $20,476.57 itemized deduction is larger and governs the calculation. Its aggregate estimate also failed to reproduce the $5,778.21 bracket tax and $1,284.46 exemption-credit subtraction."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the main AGI, charity, real-estate-tax, and exemption-credit components but applied estimated rather than traced 2026 parameters. On $139,108.47 of taxable income, bracket tax is $5,778.21 and the post-credit amount is $4,493.74, not $4,382."
-us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model replaced the required bracket-and-credit calculation with a rounded estimate of $4,250. Applying the 2026 California married-joint brackets and then subtracting $1,284.46 produces $4,493.74."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the liability from gross household income without deriving California AGI, itemized deductions, bracket tax, or exemption credits. The exact computation produces $4,493.74 rather than the coarse $4,500 estimate."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used AGI of $159,728, deductions of $21,275, and taxable income of $138,453 rather than the traced $159,585.05, $20,476.57, and $139,108.47. It also estimated both bracket tax and exemption credits instead of using $5,778.21 and $1,284.46."
-us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used taxable income of $110,811 instead of $139,108.47, reflecting an unsupported extra reduction of more than $28,000. It also asserted that no nonrefundable credits applied, omitting $1,284.46 of California exemption credits."
-us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model derived taxable income close to the traced amount but subtracted only $576 of exemption credits instead of $1,284.46. It also estimated pre-credit tax at $5,866 rather than the bracket result of $5,778.21."
-us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
-us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
-us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model set the liability to zero despite $139,108.47 of California taxable income producing $5,778.21 of tax before credits. The $1,284.46 of nonrefundable credits does not eliminate that tax and leaves $4,493.74."
-us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model incorrectly asserted that no nonrefundable credits applied, omitting $1,284.46 of California exemption credits for the two dependent children. It also treated $338 of educator expenses as a California above-the-line deduction and miscomputed bracket tax."
-us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $34,000 standard deduction instead of using the traced $20,476.57 itemized deduction and then overstated tax on its resulting $130,894 taxable income. It also applied a renter credit despite the stated income and confused that credit with the required nonrefundable exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income close to the traced amount but applied inflated California brackets, producing roughly $6,940 instead of $5,778.21 before credits. It also approximated the exemption credits rather than applying the $1,284.46 traced amount."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $4,754 joint standard deduction and ignored the larger $20,476.57 itemized deduction. It also treated exemption credits as two $73 child credits instead of applying $1,284.46 of California nonrefundable exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model reached approximately the correct AGI and itemized-deduction pathway but never completed the joint-bracket computation consistently, cycling through incompatible tax estimates. The correct taxable income of $139,108.47 yields $5,778.21 before $1,284.46 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model failed to reduce the listed gross wages by the traditional 401(k) deferrals, leaving California AGI about $3,425 too high. It then applied an overstated bracket tax to its inflated taxable income."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model treated roughly $138,000 of joint taxable income as generating about $7,700 of California tax, substantially overstating the progressive joint-bracket calculation. The traced bracket tax is $5,778.21, followed by $1,284.46 of credits."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated inflation adjustments that pushed the bracket tax down to $5,989 and assumed four exemption credits of about $450 each. The applicable computation is $5,778.21 of bracket tax less $1,284.46, not an estimated $1,800 credit."
+us,scenario_099,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model approximated taxable income at $150,000 despite the traced $159,585.05 AGI and $20,476.57 deduction yielding $139,108.47. It also failed to subtract the full $1,284.46 of exemption credits from the correctly computed bracket tax."
+us,scenario_099,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated taxable income by $26,672.47, using $112,436 instead of $139,108.47. Its unexplained $22,551 deduction total does not bridge that gap from the traced $159,585.05 AGI."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The answer implies a pre-credit tax or exemption-credit amount different from the traced figures. California tax is $5,778.21 before $1,284.46 of nonrefundable credits, yielding $4,493.74 rather than $4,665."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unsupported $7,845 answer exceeds even the traced $5,778.21 tax before credits. It failed to apply the 2026 joint brackets and then subtract $1,284.46 of nonrefundable exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI by $24,416.05 and overstated itemized deductions by $5,661.43, reducing taxable income to $109,031 instead of $139,108.47. It also used only about $600 of exemption credits instead of $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The unexplained $2,570 result implies a major understatement of the bracket tax or an excessive credit subtraction. The required sequence is $5,778.21 of tax less $1,284.46 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The answer understated post-credit liability by $365.74 without identifying any additional allowable nonrefundable credit. The traced bracket tax and exemption credits produce $4,493.74."
+us,scenario_099,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer understated the liability by $437.74, reflecting an incorrect bracket computation or excessive exemption credits. The applicable amounts are $5,778.21 before credits and $1,284.46 of credits."
+us,scenario_099,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly used a $6,566 standard deduction and asserted that charitable contributions are deductible only to the extent exceeding that deduction. California instead allows the $20,476.57 itemized deduction in place of the standard deduction, and the $338 educator expense is not itself a tax credit."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unsupported $6,713 result is above the traced $5,778.21 tax even before credits. It omitted or misapplied the progressive joint brackets and the $1,284.46 nonrefundable exemption-credit subtraction."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The submitted $4,931 conflicts with its explanation, which states $4,931 but supplies no computation. The correct bracket-and-credit sequence yields $4,493.74, so its number reflects an overstated bracket tax or understated exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model improperly excluded health premiums and other amounts from California AGI, reducing it to $134,323, then reported taxable income of $101,811. The traced AGI is $159,585.05 and taxable income is $139,108.47; its stated $6,882 tax is also inconsistent with its own much lower taxable-income estimate."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model says it used the standard deduction, but the household's $20,476.57 itemized deduction controls. It also failed to reproduce the $5,778.21 bracket tax less $1,284.46 of exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model identified the correct general itemized-deduction and exemption-credit pathway but used approximate inputs and returned $111.74 too little. Exact 2026 brackets and credits produce $5,778.21 less $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The unsupported estimate understates the result by $243.74. It did not apply the exact $5,778.21 bracket tax and $1,284.46 nonrefundable exemption credits."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model rounded the liability to $4,500 from gross-income intuition rather than computing the traced brackets and credits. Exact application yields $4,493.74."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model reached taxable income close to the traced $139,108.47 but approximated both bracket tax and credits. The exact amounts are $5,778.21 and $1,284.46, producing $4,493.74 rather than $4,600."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used $21,275 of itemized deductions and therefore $138,791 of taxable income, while the trace allows $20,476.57 and yields $139,108.47. Its approximated bracket tax and $1,246 credit also differ from the exact $5,778.21 and $1,284.46."
+us,scenario_099,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to $110,811 instead of $139,108.47 and then asserted that no nonrefundable credits apply. The household receives $1,284.46 of California exemption credits for the filers and dependents."
+us,scenario_099,state_income_tax_before_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model allowed only $576 of exemption credits, understating them by $708.46. California applies $1,284.46 of nonrefundable exemption credits after the $5,778.21 bracket tax."
+us,scenario_099,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_099,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_099,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model set the liability to zero despite $139,108.47 of California taxable income. The bracket tax is $5,778.21, and the $1,284.46 of nonrefundable credits cannot eliminate it."
+us,scenario_099,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly claimed that taking the federal standard deduction forces use of California's standard deduction. California itemization produces a $20,476.57 deduction and $139,108.47 of taxable income, not roughly $147,800."
+us,scenario_099,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model treated the $338 educator expense as a California above-the-line deduction and then applied no nonrefundable exemption credits. The traced computation uses $159,585.05 of AGI and subtracts $1,284.46 of exemption credits after calculating $5,778.21 of tax."
+us,scenario_099,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model invented a $34,000 California standard deduction and used it instead of the traced $20,476.57 itemized deduction. It also applied an inapplicable renter credit and overstated the bracket tax on its stated taxable income."
 us,scenario_099,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_099,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -5862,30 +6154,32 @@ us,scenario_100,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_out
 us,scenario_100,child2_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model described a child aged 5 as “under 5,” which is the exact categorical error. Falling below the income threshold does not establish WIC eligibility for a child who has aged out."
 us,scenario_100,child2_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model interpreted “up to age 5” as including children who are already 5. WIC child categorical eligibility ends at the fifth birthday, regardless of the household's low income."
 us,scenario_100,child2_wic_eligible,qwen3.8-max,llm_error,categorical_eligibility,False,"The model included age 5 in the WIC child category and then relied on the income limit. Because the child is not under five, the categorical test fails before income can establish eligibility."
-us,scenario_100,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $2,878 from a $2,366 EITC and $512 refundable CTC, then submitted $6,425 instead. Its numeric output discarded its own completed calculation."
-us,scenario_100,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model correctly calculated refundable CTC as $512.25, then replaced it with an unsupported $834 approximation. Refundable CTC is limited to 15% of $5,915 minus $2,500, yielding $512.21 in the engine calculation."
-us,scenario_100,federal_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model arbitrarily reduced the EITC from its correctly calculated $2,366 phase-in amount to $1,906 by invoking an inapplicable smaller-income test. In the phase-in range, lower AGI does not reduce the credit computed from $5,915 of earned income."
-us,scenario_100,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived approximately $2,366 of EITC and $512 of refundable CTC, which sum to approximately $2,878, but submitted $2,330. Its unexplained downward adjustment contradicts both component calculations."
-us,scenario_100,federal_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the separately listed $710 FLSA overtime premium to the annual gross-wage input when computing EITC. The prompt states that gross wages are annual totals including overtime, so earned income remains $5,915 and the EITC is $2,365.89."
-us,scenario_100,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model correctly computed a $2,366 EITC and $512 refundable CTC, then substituted an EITC-table-based total of $6,295. At $5,915 of earnings, the household remains in the 40% EITC phase-in range and does not receive the maximum credit."
-us,scenario_100,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted retirement contributions from wages to reduce EITC earned income to $928 and eliminate refundable CTC. The engine uses the $5,915 wage amount for both credits, producing $2,365.89 of EITC and $512.21 of refundable CTC."
-us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied a $3,000 refundable-CTC earnings threshold. The applicable engine threshold is $2,500, so the refundable CTC is $512.21 rather than $437.25."
-us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated low income as qualifying the household for full EITC and refundable CTC amounts. Both credits phase in with earnings here: the components are $2,365.89 and $512.21, not their maximum amounts."
-us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated employer-sponsored insurance premiums as pre-tax deductions that erase the explicitly stated $5,915 of annual gross wages. Those premiums do not reduce the benchmark wage input to zero for EITC and refundable-CTC calculations."
-us,scenario_100,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced earned income to zero using the listed health-insurance premiums. The $5,915 gross-wage input generates a $2,365.89 EITC and $512.21 refundable CTC."
-us,scenario_100,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $3,000 earnings threshold for refundable CTC instead of $2,500. That omitted $75 of the earnings-based refundable credit before exact engine rounding."
-us,scenario_100,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model's submitted value was zero even though its explanation expressly calculated $2,803.25 and said the household qualified for both credits. The numeric field therefore contradicted its own stated answer and the required same-value contract."
-us,scenario_100,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model awarded the $1,700-per-child refundable-CTC ceiling without applying the earnings limitation of 15% above $2,500. It also computed EITC from reduced AGI instead of the $5,915 earned-income phase-in base."
-us,scenario_100,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated very low income as preventing refundable credits. Two qualifying children and $5,915 of earnings trigger both the EITC phase-in and the earnings-based refundable CTC."
-us,scenario_100,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable credit was triggered. The household's positive earnings and two qualifying children generate $2,365.89 of EITC and $512.21 of refundable CTC."
-us,scenario_100,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model incorrectly assigned a zero EITC phase-in amount despite $5,915 of earned income. The two-child 40% phase-in produces $2,365.89, with another $512.21 from refundable CTC."
-us,scenario_100,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted the traditional 401(k) contribution from the benchmark's gross-wage amount when defining EITC earned income. Using the full $5,915 also places earnings above the applicable $2,500 refundable-CTC threshold."
-us,scenario_100,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model treated wages after the traditional 401(k) deferral as the EITC and refundable-CTC earned-income base. The engine uses $5,915, producing both a $2,365.89 EITC and a positive $512.21 refundable CTC."
-us,scenario_100,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model deducted both traditional 401(k) and IRA contributions from wages to obtain $1,876 of EITC earned income. Those deductions do not replace the $5,915 earned-income base, which also exceeds the refundable-CTC threshold."
-us,scenario_100,federal_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model reduced EITC earned income to W-2 Box 1 wages after the traditional 401(k) deferral and consequently eliminated refundable CTC. The benchmark calculation uses the stated $5,915 gross wages, yielding both refundable components."
-us,scenario_100,federal_refundable_credits,minimax-m3,llm_error,other,False,The model's reasoning asserted substantial positive EITC and refundable CTC amounts but submitted zero. The numeric output contradicts its own eligibility analysis and final-credit discussion.
-us,scenario_100,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model treated the per-child refundable-CTC cap as an automatic award and later substituted an unsupported $3,730 EITC. Refundable CTC is instead constrained to 15% of earnings above $2,500, and EITC equals the 40% phase-in amount at this income."
-us,scenario_100,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly capped EITC and refundable CTC at tax liability or zero. These are refundable credits: positive earned income generates $2,365.89 of EITC and $512.21 of refundable CTC even with no pre-credit income-tax liability."
+us,scenario_100,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived approximately $2,878 from a $2,366 EITC and $512 refundable CTC, then submitted $6,425 instead. Its final value discards its own completed computation."
+us,scenario_100,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model correctly computed the refundable CTC earnings limit as $512.25, then replaced it with an unsupported $834 approximation. Adding the $2,365.89 EITC to the earnings-limited $512.21 refundable CTC yields $2,878.10."
+us,scenario_100,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model stated that $5,915 at the 40% EITC phase-in rate produces about $2,366, then inexplicably reduced the EITC to $1,906 under an AGI test. A lower AGI does not reduce a phase-in EITC below the amount computed from $5,915 of earned income."
+us,scenario_100,federal_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model derived both correct components—about $2,366 of EITC and $512 of refundable CTC—but submitted $2,330 after an unexplained reduction. Those components sum to $2,878.10, not $2,330."
+us,scenario_100,federal_refundable_credits,claude-opus-5,llm_error,payroll_tax_base,False,"The model incorrectly added the separately listed $710 FLSA overtime premium to the $5,915 annual gross wages even though gross wages already include overtime pay. EITC earned income remains $5,915, producing $2,365.89 rather than $2,905."
+us,scenario_100,federal_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived a $2,366 EITC and $512 refundable CTC, then abandoned that calculation and substituted $6,295 based on an inapplicable full-credit-table intuition. At $5,915 the EITC is still in its 40% phase-in range, so the two refundable components total $2,878.10."
+us,scenario_100,federal_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reduced EITC and refundable-CTC earned income to $928 using retirement deductions. The applicable earned-income base is $5,915, which generates $2,365.89 of EITC and $512.21 of refundable CTC."
+us,scenario_100,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $3,000 refundable-CTC earnings threshold. The applicable threshold is $2,500, so the refundable CTC is $512.21 rather than $437.25."
+us,scenario_100,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model treated low income as qualifying the household for full refundable credits. At $5,915, both credits are earnings-limited: EITC is $2,365.89 and refundable CTC is $512.21, totaling $2,878.10."
+us,scenario_100,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted the reported employer-sponsored insurance premiums from wages and set earned income to zero. The annual gross-wage input is $5,915 and is not erased by that separately reported premium amount, so it produces both EITC and refundable CTC."
+us,scenario_100,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced earned income to zero using the health-insurance premium input. The $5,915 gross-wage amount remains the credit earnings base, yielding $2,365.89 of EITC and $512.21 of refundable CTC."
+us,scenario_100,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used a $3,000 refundable-CTC earnings threshold instead of $2,500. That understated the refundable CTC by $74.96 and produced $2,803.25 instead of $2,878.10."
+us,scenario_100,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The explanation calculated $2,803.25, but the model submitted zero. It therefore discarded even its own positive EITC and refundable-CTC determination; applying the correct $2,500 threshold gives $2,878.10."
+us,scenario_100,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model awarded the $1,700 per-child refundable cap without first applying the refundable CTC's family-level earnings limitation. It also based EITC on reduced AGI rather than $5,915 of earned income; the correct components are $512.21 and $2,365.89."
+us,scenario_100,federal_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated very low earnings as disqualifying. Two qualifying children and $5,915 of earned income place the household in the EITC phase-in and above the refundable-CTC earnings threshold, generating $2,878.10."
+us,scenario_100,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that no refundable credit was triggered. The household has two qualifying children and positive earnings above the refundable-CTC threshold, producing $2,365.89 of EITC plus $512.21 of refundable CTC."
+us,scenario_100,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly asserted that $5,915 yields no EITC phase-in amount. With two qualifying children, the 40% phase-in produces $2,365.89, and the refundable CTC adds $512.21."
+us,scenario_100,federal_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model reduced EITC earned income by the desired traditional 401(k) contribution and then used the obsolete $3,000 refundable-CTC threshold. The applicable earnings base is $5,915 and the threshold is $2,500, producing both $2,365.89 of EITC and $512.21 of refundable CTC."
+us,scenario_100,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model correctly computed the EITC but applied a $3,000 refundable-CTC earnings threshold. Using the applicable $2,500 threshold raises the refundable CTC from about $437 to $512.21."
+us,scenario_100,federal_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model treated the desired traditional 401(k) contribution as reducing the EITC and refundable-CTC earned-income base to $2,056. The credits use $5,915 here, so EITC is $2,365.89 and earnings exceed the $2,500 refundable-CTC threshold."
+us,scenario_100,federal_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model subtracted both desired traditional retirement contributions from earned income and used $1,876 for the EITC and refundable CTC. The applicable earned-income amount is $5,915, producing $2,365.89 of EITC and $512.21 of refundable CTC."
+us,scenario_100,federal_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model reduced earned income to W-2 Box 1 after the desired 401(k) contribution and therefore eliminated the refundable CTC. The benchmark uses $5,915 as earned income, placing the household above the $2,500 threshold and generating both credit components."
+us,scenario_100,federal_refundable_credits,minimax-m3,llm_error,other,False,"The model's reasoning asserted substantial positive EITC and refundable CTC but submitted zero, so its final value contradicts its own eligibility conclusion. At $5,915, the credits are earnings-limited to $2,365.89 and $512.21 rather than full-credit amounts."
+us,scenario_100,federal_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model applied the 15% refundable-CTC earnings formula separately to each child. That earnings limitation is computed once for the tax unit, so refundable CTC is $512.21 total, not $512.25 per child."
+us,scenario_100,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model awarded a full per-child refundable CTC and later substituted an unsupported $3,730 EITC despite correctly identifying the 40% phase-in calculation. The refundable CTC is constrained by 15% of family earnings above $2,500 to $512.21, and EITC is $2,365.89."
+us,scenario_100,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly capped refundable credits by income-tax liability and treated low earnings as producing zero. EITC and the refundable portion of CTC are payable beyond tax liability; $5,915 of earnings generates $2,878.10 in total."
 us,scenario_100,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model asserted that positive support was not indicated instead of deriving program receipt and income from the supplied household facts under the assumed-take-up instruction. It missed both automatic categorical eligibility through $8,625.89 of annual SNAP and the independent income pathway, under which countable income equals 44% of the federal poverty guideline and is below the 130% free-meal threshold."
 us,scenario_100,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model evaluated the head only under an expansion/disability pathway and missed the parent/caretaker category, whose Montana income limit is 100% FPL and which the head satisfies at 0.07 × FPL. It also used an erroneous $3,627 annual 138%-FPL threshold and incorrectly imposed an asset limit on MAGI-based Medicaid eligibility."
 us,scenario_100,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored the parent/caretaker pathway established by the head living with two dependent children. Under that category, the head’s MAGI of 0.07 × FPL is below Montana’s 100%-FPL limit, and neither separate disability-based expansion nor lack of other coverage is required."
@@ -5905,130 +6199,138 @@ us,scenario_100,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model identi
 us,scenario_100,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model miscalculated 6.2% of $5,915 as $367.13 instead of about $366.73, then replaced its own $452.91 subtotal with an unsupported rounded value of $477. The engine’s component calculations are $366.71 for Social Security and $85.76 for Medicare, totaling $452.48."
 us,scenario_100,reduced_price_school_meals_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated income below 185% of the poverty guideline as sufficient for reduced-price meals and ignored that income at or below 130% qualifies for free meals. At 44% of the guideline, reinforced by categorical eligibility, the household receives the mutually exclusive FREE tier rather than reduced-price support."
 us,scenario_100,reduced_price_school_meals_eligible,claude-opus-4.8,llm_error,thresholds_rates,False,"The model incorrectly asserted that $5,915 was above 130% of the poverty guideline for a four-person household, even though the engine ratio is 44%. It therefore placed the household in the 130%-185% reduced-price band instead of the FREE tier, while also overlooking categorical eligibility for free meals."
-us,scenario_100,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used wages alone as gross income and substituted a $94 medical deduction for the $453.64 excess shelter deduction. The correct monthly calculation starts from $998.31 gross income, reaches $237.09 net income, and subtracts a $71.10 contribution from the applicable monthly maximum."
-us,scenario_100,snap,claude-haiku-4.5,llm_error,other,False,"The model established eligibility but never performed the SNAP allotment calculation; its $1,908 figure has no derivation from a three-person maximum allotment and 30% of net income. Monthly allotments of about $713.90 and $733.60 sum to $8,625.89."
-us,scenario_100,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used wages alone, invented a $141 monthly medical deduction, and omitted the $453.64 excess shelter deduction. It then abandoned its own $9,108 calculation for an unsupported $7,320 estimate instead of using $237.09 net income."
-us,scenario_100,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model calculated net income from wages alone and treated medical expenses as the remaining adjustment, omitting the traced $998.31 gross income and $453.64 excess shelter deduction. Those inputs produce $237.09 net income and a $71.10 monthly household contribution."
-us,scenario_100,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,The model used only annual wages to characterize net income as very low and never applied the full $998.31 monthly gross-income calculation or the $453.64 excess shelter deduction. It therefore estimated the contribution instead of computing 30% of $237.09.
-us,scenario_100,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted income included in the $998.31 monthly gross amount, set shelter costs to zero, and replaced the $453.64 excess shelter deduction with a medical deduction. It also used estimated benefit parameters rather than the $785 and later $802.60 monthly maxima."
-us,scenario_100,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model treated the employer-sponsored insurance premium as a deductible medical expense and asserted zero net income, while the traced deductions yield $237.09. It also annualized an asserted $766 monthly maximum to $13,700, an arithmetic result incompatible with twelve months."
-us,scenario_100,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model included TANF in gross income but deducted only $46.67 of medical costs and omitted the $453.64 excess shelter deduction. That inflated net income to $602.21 instead of $237.09 and overstated the household contribution.
-us,scenario_100,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used wages alone and a $140 medical deduction, producing $54 net income instead of the traced $237.09 after the standard, earned-income, and excess-shelter deductions. It consequently understated the 30% contribution and used an estimated maximum."
-us,scenario_100,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer equals twelve months of the $784-style maximum and omits the $71.10 monthly contribution from $237.09 net income. It also fails to incorporate the later-month change to the $802.60 maximum.
-us,scenario_100,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model asserted a near-maximum benefit but used an annual maximum far below the applicable three-person allotments of $785 and later $802.60. Applying the $71.10 monthly contribution and summing the monthly amounts yields $8,625.89."
-us,scenario_100,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorporated TANF but did not apply the traced $453.64 excess shelter deduction and therefore overstated net income and the benefit reduction. The correct net income is $237.09, producing a $71.10 monthly contribution."
-us,scenario_100,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,The model based the calculation on $492.92 of monthly wages rather than the traced $998.31 gross income and did not use the $453.64 excess shelter deduction. This produced the wrong net income and household contribution.
-us,scenario_100,snap,gemini-3.7-flash,llm_error,other,False,"The explanation addresses eligibility but supplies no allotment calculation, and it states value = 9072 while submitting 9216. The annual amount must be obtained by summing the monthly $713.90 and $733.60 allotments to $8,625.89."
-us,scenario_100,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the employer-sponsored insurance premium and other household medical expenses as deductions that eliminate net income. The traced allowable deductions total $761.21, including $453.64 of excess shelter costs, leaving $237.09 rather than zero."
-us,scenario_100,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model assigned an annual maximum of $12,588, which is incompatible with the applicable three-person monthly maxima of $785 and later $802.60. It also omitted the $71.10 monthly contribution required by $237.09 net income."
-us,scenario_100,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly treated absent user-entered housing details as making SNAP zero. The household passes the eligibility tests, including categorical eligibility through TANF non-cash status, and the engine applies a $453.64 excess shelter deduction."
-us,scenario_100,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated reported medical and insurance expenses as eliminating countable income and awarded a full maximum. The allowable traced deductions instead leave $237.09 net income, so $71.10 is subtracted from each monthly maximum."
-us,scenario_100,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model treated low earnings as producing zero net income and awarded twelve full $785 allotments. The correct net income is $237.09, requiring a $71.10 monthly contribution, with the maximum rising to $802.60 in later months."
-us,scenario_100,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used medical expenses to reduce SNAP net income to zero and therefore awarded the full maximum. The traced deductions include $453.64 of excess shelter costs and leave $237.09 net income, not zero."
-us,scenario_100,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model mentions SSI and medical deductions instead of following the traced gross-income and excess-shelter calculation. Using $998.31 gross income, $761.21 of deductions, and the resulting $71.10 contribution produces $8,625.89."
-us,scenario_100,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model returned zero without applying any eligibility or benefit rule. The household satisfies categorical eligibility through TANF non-cash status and all income and asset tests, so positive monthly allotments must be calculated."
-us,scenario_100,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used wages alone, an outdated $198 standard deduction, and a medical deduction instead of the traced $209 standard deduction and $453.64 excess shelter deduction. It therefore failed to derive $237.09 monthly net income and the $71.10 contribution."
-us,scenario_100,snap,grok-build-0.1,llm_error,period_annualization,False,The model returned a single monthly maximum as the annual output and did not subtract the household contribution. The benchmark requires summing twelve monthly allotments after subtracting $71.10 from each applicable maximum.
-us,scenario_100,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model used wages and a disabled-member medical deduction to estimate net income near $50. The traced gross income and allowable deductions instead yield $237.09, so the monthly contribution is $71.10 rather than about $15."
-us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the required output was missing."
-us,scenario_100,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,The model incorrectly treated premiums and medical expenses as reducing net SNAP income to zero and awarded twelve full $785 allotments. The traced allowable deductions leave $237.09 net income and require a $71.10 monthly contribution.
-us,scenario_100,snap,minimax-m3,llm_error,other,False,"The model first described a near-maximum benefit, then reduced $766 to $377 per month without a valid 30%-of-net-income calculation. With $237.09 net income, the required reduction is $71.10 per month, not $389."
-us,scenario_100,snap,qwen-3.7-max,llm_error,period_annualization,False,"The model double-counted the $710 overtime premium on top of gross wages even though gross wages already include overtime. It also claimed $766 per month annualizes to $12,444; twelve such months equal $9,192 before any household contribution."
-us,scenario_100,snap,qwen3.8-max,llm_error,asset_resource,False,"The model incorrectly denied SNAP for excess resources. The household's $2,800 of counted assets is within the applicable limit, and categorical eligibility through TANF non-cash status also supports eligibility."
-us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head’s generic `is disabled` fact as satisfying SSI’s program-specific aged, blind, or disabled criterion, even though `is_ssi_aged_blind_disabled=False`. It therefore performed an inapplicable earned-income calculation instead of assigning $0 to every categorically ineligible individual; its submitted $8,285 also contradicts every amount derived in its explanation."
-us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed that the head’s generic disability status established SSI categorical eligibility. Because the head and both children have `is_ssi_aged_blind_disabled=False`, no earned-income reduction calculation applies and each person’s SSI is $0."
-us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the head as an SSI-eligible disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails the program-specific aged, blind, or disabled test, as do both children, so the maximum benefit and income exclusions never enter the computation."
-us,scenario_100,ssi,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model explicitly classified the head as SSI eligible from the generic disability fact. SSI categorical eligibility is false for the head and both children, so no federal maximum or earned-income exclusion produces a payable benefit."
-us,scenario_100,ssi,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model assumed the head qualified for SSI and calculated a federal maximum less countable earned income. The head’s `is_ssi_aged_blind_disabled` status is false, so SSI stops at categorical ineligibility and equals $0 before income counting."
-us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The model equated the generic disability input with SSI disability eligibility and applied the $20 and $65 income exclusions plus the one-half earned-income rule. Those exclusions are irrelevant because no household member satisfies SSI’s aged, blind, or disabled criterion."
-us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable SSI value or explanation. The required output was a numeric value of 0 supported by the fact that all three individuals fail SSI’s categorical eligibility test.
-us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the head’s generic disability fact as SSI categorical qualification and proceeded to income exclusions and an unsupported IRWE adjustment. The head has `is_ssi_aged_blind_disabled=False`, so SSI is $0 before any earnings, overtime, exclusions, or work-expense adjustments are considered."
-us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model used a 10% Montana EITC rate instead of 20% and incorrectly reduced EITC earned income for the desired traditional 401(k) contribution. The federal EITC is $2,365.89 and Montana refunds 20%, yielding $473.18."
-us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Montana has no refundable EITC. This household qualifies for the refundable Montana EITC equal to 20% of its $2,365.89 federal EITC."
-us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model correctly identified a federal EITC near $2,366 but applied an obsolete 10% Montana rate and then submitted an amount inconsistent with even that calculation. The applicable state rate is 20%, producing $473.18."
-us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model acknowledged that the household has a positive federal EITC and that Montana provides a refundable state EITC, then discarded that credit and submitted zero. Applying Montana’s 20% rate to $2,365.89 yields $473.18."
-us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly treated Montana’s EITC as nonrefundable. In 2026 it is refundable at 20% of the federal EITC, so zero state tax liability does not prevent the $473.18 payment."
-us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model correctly computed the federal EITC at approximately $2,366 but applied a 10% Montana rate. The 2026 rate is 20%, making the refundable credit $473.18."
-us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly classified the Montana EITC as nonrefundable and therefore eliminated it when tax liability was zero. The credit is refundable at 20% of the $2,365.89 federal EITC."
-us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used both the wrong Montana rate, 10% instead of 20%, and an unsupported federal EITC of $371. With two qualifying children, the 40% phase-in on $5,915 produces $2,365.89, whose 20% state share is $473.18."
-us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly obtained a federal EITC of about $2,366 but used a 10% Montana EITC percentage. Montana’s 2026 percentage is 20%, yielding $473.18."
-us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Montana’s refundable EITC. The household’s two qualifying children and $5,915 of earned income generate a $2,365.89 federal EITC and a $473.18 Montana credit."
-us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly stated that Montana lacks an applicable refundable EITC and also treated the household as having zero earned income despite $5,915 of wages. The applicable 20% state EITC is $473.18."
-us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly set the federal EITC to zero. Two qualifying children and $5,915 of earned income produce a $2,365.89 federal EITC, which generates a $473.18 Montana refundable EITC."
-us,scenario_100,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model used a 3% Montana EITC rate instead of 20%, and its submitted zero also contradicted its stated $70.98 calculation. The correct state calculation is 20% of $2,365.89, or $473.18."
-us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana has no individual income tax. Montana imposes an individual income tax and provides a refundable EITC equal to 20% of the federal credit, producing $473.18 here."
-us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model omitted the refundable Montana EITC triggered by the household’s wages and two qualifying children. The federal EITC is $2,365.89, and its 20% Montana counterpart is $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly concluded that this low-income profile produces no refundable Montana credit. It qualifies for a state EITC of 20% of the $2,365.89 federal EITC, equal to $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated the federal EITC as about $4,207 and then applied an obsolete 10% Montana rate. In the phase-in range, $5,915 times the 40% federal rate yields $2,365.89, and Montana pays 20%, or $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model correctly estimated the federal EITC at $2,366 but applied a 10% Montana rate. The applicable rate is 20%, producing $473.18."
-us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied a 10% rather than 20% Montana EITC rate and added a nonexistent $1,200 refundable young-child credit for the five-year-old. The only applicable state refundable credit is the $473.18 Montana EITC."
-us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model applied a 10% rather than 20% Montana EITC rate and incorrectly added a $1,200 young-child credit. Montana refundable credits for this household consist solely of the $473.18 EITC."
-us,scenario_100,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted Montana’s refundable EITC without applying the household’s federal EITC eligibility. The $2,365.89 federal credit generates a $473.18 state credit at Montana’s 20% rate."
-us,scenario_100,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly stated that Montana offers no state EITC. Montana’s refundable EITC equals 20% of the federal EITC and pays $473.18 to this household.
-us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,"The model omitted the applicable refundable Montana EITC. Two qualifying children and $5,915 of wages produce a $2,365.89 federal EITC and a $473.18 state credit."
-us,scenario_100,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model invented a $1,200 refundable Montana child credit, used a 10% state EITC rate, and understated the federal EITC as about $750. The sole applicable refundable credit is 20% of the $2,365.89 federal EITC, or $473.18."
-us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model incorrectly stated that Montana offers no refundable EITC and focused only on the unavailable property-tax circuit breaker. This household instead receives a $473.18 refundable Montana EITC based on its federal credit.
-us,scenario_100,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model correctly computed the federal EITC at about $2,366 but used a 10% Montana rate. The 2026 rate is 20%, so the state refundable credit is $473.18."
-us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model incorrectly asserted that Montana has no state income tax. Montana has an individual income tax and a refundable EITC equal to 20% of the federal credit, yielding $473.18."
-us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly stated that Montana has no earned income tax credit and treated zero tax liability as eliminating refundable credits. The household qualifies for a refundable Montana EITC of $473.18.
-us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted the Montana refundable EITC that applies to the household’s $2,365.89 federal EITC. Montana’s 20% state credit equals $473.18."
-us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"The model correctly recognized that countable income was below the payment standard, then contradicted that calculation by declaring the grant zero. It also substituted guessed $588/$633 monthly standards and a $200-plus-25% disregard instead of applying the 2026 Montana benefit parameters that yield $6,064.96."
-us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"The model used the wrong assistance-unit size, calling the head and two children a family of four, and imposed a guessed $2,000 asset limit. Under the applicable Montana resource and income rules, the $2,800 bank balance and $5,915 annual wages do not disqualify this three-person unit."
-us,scenario_100,tanf,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that the income test produced zero without performing a consistent benefit calculation, even though it acknowledged low earnings, assumed take-up, and assets within the limits. Applying Montana’s actual 2026 eligibility and payment rules produces a positive $6,064.96 grant."
-us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"The model treated an approximate $595 monthly figure as the payable grant and effectively awarded it for all 12 months without the exact Montana calculation. The applicable 2026 payment formula yields $6,064.96, not $7,140."
-us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"The model declared that PolicyEngine’s modeled rules produce no benefit despite the qualifying parent, two dependent children, and income well below Montana’s limits. It omitted the eligibility and benefit-formula steps that produce $6,064.96."
-us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"The model imposed an approximately $1,000 liquid-asset limit and used it to disqualify the household. Montana’s applicable 2026 resource treatment does not make the $2,800 bank balance disqualifying, so the model never reached the positive benefit calculation."
-us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used an obsolete or inapplicable $327 monthly maximum and an invented hybrid earned-income disregard. The 2026 Montana payment parameters produce $6,064.96 for this family, substantially above the model’s $3,924 estimate."
-us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model substituted a $605 maximum and a $200-plus-50% earnings disregard, then subtracted its resulting countable income dollar for dollar. Those are not the parameter combination governing this 2026 Montana calculation, which yields $6,064.96."
-us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a guessed $588 payment standard and the formula 75% of earnings above $200. It then introduced an unsupported rounding adjustment from $4,416 to $4,419; the actual Montana parameters yield $6,064.96."
-us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model asserted failure of unspecified state requirements despite a qualifying three-person family with very low annual income. It omitted Montana’s actual eligibility pathway and positive benefit computation.
-us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model invented SSI receipt and used it to deny TANF even though no SSI receipt was listed and unlisted benefit receipt had to be treated as zero. The household’s stated facts qualify it for the $6,064.96 TANF amount."
-us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a guessed $200-plus-50% earned-income disregard and an unspecified grant standard. Montana’s exact 2026 parameters instead produce an annual benefit of $6,064.96."
-us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model reduced the grant using an unspecified earned-income-disregard calculation that produced $5,298.48. That calculation does not implement the applicable 2026 Montana benefit parameters, which yield $6,064.96."
-us,scenario_100,tanf,gemini-3.7-flash,llm_error,other,False,"The model’s submitted answer was zero while its explanation ended with value = 4824 and stated that the household qualifies. This violates the required agreement between the numeric value and explanation, and neither figure implements the $6,064.96 calculation."
-us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,The model improperly added the separately listed FLSA overtime premium to gross wages even though the prompt states gross wages already include overtime. It also treated the excluded vehicle as potentially countable and incorrectly concluded that low earnings exceeded the payment standard.
-us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated absence of an explicitly listed TANF payment as evidence that no benefit was payable, despite the instruction to assume take-up. It failed to calculate eligibility and the Montana benefit amount from the supplied household facts."
-us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model incorrectly required an additional TANF eligibility trigger beyond a caretaker parent, two dependent children, and qualifying income. Those listed facts establish the relevant family pathway and lead to a positive $6,064.96 benefit."
-us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"The model guessed that approximately $150 of monthly earnings was countable and used a $588 payment standard. Those approximations do not match Montana’s 2026 payment calculation, understating the annual grant by $808.96."
-us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model used an estimated $359.50 monthly award without identifying or applying the governing Montana standard and income treatment. The exact state formula yields $6,064.96 annually."
-us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the same guessed $588 standard and $200-plus-25% disregard calculation that produces $4,419.75. That historical or inapplicable parameter combination is not Montana’s 2026 benchmark formula."
-us,scenario_100,tanf,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model invented countable SSI income even though no SSI receipt was listed and all unlisted numeric inputs were zero. Wages alone do not eliminate the grant, and the correct computation produces $6,064.96."
-us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,The model defaulted TANF to zero because no precomputed TANF amount appeared in the inputs. It failed to apply Montana eligibility and benefit rules to the qualifying household facts.
-us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"The model used a $588 monthly standard and a $200-plus-25% earnings disregard, producing about $368 per month. Those are not the applicable 2026 Montana parameters, and its own multiplication also rounds away from the stated intermediate amount."
-us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"The model paired a guessed $675 maximum with a $200-plus-25% disregard, then subtracted approximately $220 monthly. This incorrect parameter combination understates the Montana annual benefit, which is $6,064.96."
-us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"The model relied on an approximate $588 maximum and a $200-plus-25% earnings disregard rather than Montana’s exact 2026 rules. That shortcut produces about $4,420 instead of $6,064.96."
+us,scenario_100,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used wages alone as gross income, deducted $93.67 of medical expenses, and omitted the $453.64 excess-shelter deduction. The required calculation starts from $998.31 gross and yields $237.09 net income and a $71.10 household contribution."
+us,scenario_100,snap,claude-haiku-4.5,llm_error,other,False,The model established eligibility but never calculated an allotment from net income and the three-person maximum. Its unexplained $159 monthly amount omits the derivation yielding monthly benefits near $713.90 and $733.60.
+us,scenario_100,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used wages alone, invented a $141 monthly medical deduction, and then abandoned its own calculated $9,108 estimate for $7,320 without a computation. It omitted the countable income and excess-shelter deduction that produce $237.09 of net income."
+us,scenario_100,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,The model based net income on wages alone and did not apply the traced $453.64 excess-shelter deduction or full $998.31 gross income. It also used a flat estimated maximum instead of the monthly $785 and $802.60 maximum allotments.
+us,scenario_100,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model treated gross income as only $5,915 of wages and estimated an unspecified small net income. It failed to calculate $237.09 of net income from $998.31 gross and the three traced deductions."
+us,scenario_100,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used wages alone, substituted a medical deduction for the $453.64 excess-shelter deduction, and cycled through incorrect historical maximum allotments. The correct monthly contribution is $71.10, applied against maximums of $785 and later $802.60."
+us,scenario_100,snap,claude-sonnet-5,llm_error,period_annualization,False,"The model incorrectly treated the large employer-sponsored premium as a SNAP medical deduction and reduced net income to zero. It then stated a maximum near $766 per month but annualized it to $13,700, which is arithmetically inconsistent with twelve months."
+us,scenario_100,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model included TANF but used the wrong TANF amount, an incorrect $204 standard deduction, an invented medical deduction, and no excess-shelter deduction. Those errors produced $602.21 net income instead of $237.09."
+us,scenario_100,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,The model used wages alone and deducted $140 of medical costs while omitting the $453.64 excess-shelter deduction. This reduced its net income to $54 instead of the traced $237.09.
+us,scenario_100,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,The answer equals twelve months of the $784 maximum and therefore treats the household as receiving a flat maximum allotment. It omits the $71.10 monthly household contribution and the later change in the maximum to $802.60.
+us,scenario_100,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,The model labeled the benefit near-maximum but used an amount averaging only $510.67 per month. It failed to apply the actual three-person maximums of $785 and $802.60 minus the $71.10 contribution.
+us,scenario_100,snap,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,The model incorporated TANF but did not calculate the traced $998.31 gross income or the $453.64 excess-shelter deduction. Its implied contribution is therefore too large relative to the correct $71.10 monthly contribution.
+us,scenario_100,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model based the calculation on $492.92 of wages and unspecified standard and medical deductions. It omitted the full $998.31 gross-income base and the traced excess-shelter deduction, so it did not reach $237.09 of net income."
+us,scenario_100,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model supplied no allotment or net-income computation and its explanation's value of $9,072 conflicts with its submitted $9,216. Neither figure reflects the changing monthly maximums minus the $71.10 contribution."
+us,scenario_100,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $20,644 employer-sponsored insurance premium and other household members' expenses as the disabled head's deductible medical expenses, forcing net income to zero. It also replaced the actual $785 and $802.60 monthly maximums with a flat $800 estimate."
+us,scenario_100,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated the household as receiving a maximum allotment but used $1,049 per month, which is not the three-person SNAP maximum. It also omitted the $71.10 monthly contribution from $237.09 of net income."
+us,scenario_100,snap,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the absence of listed rent and utilities as making SNAP zero. PolicyEngine applies a $453.64 excess-shelter deduction from the supplied household inputs, and the household qualifies for positive benefits."
+us,scenario_100,snap,gpt-5.5,llm_error,taxable_income_or_deductions,False,The model incorrectly deducted reported health and insurance expenses until net income reached zero. The applicable deductions instead leave $237.09 of net income and require a $71.10 monthly contribution.
+us,scenario_100,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,The model paid the flat $785 maximum for all twelve months. It omitted the $71.10 contribution and the later increase of the maximum allotment to $802.60.
+us,scenario_100,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly used medical premiums and expenses to reduce SNAP net income to zero and paid the maximum allotment. The traced deductions leave $237.09 net income, so 30% must reduce each month's maximum."
+us,scenario_100,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model referenced earnings, SSI, and medical deductions without calculating the traced gross or net income. Its $533.33 monthly average is inconsistent with the $785 and $802.60 maximums less the $71.10 contribution."
+us,scenario_100,snap,grok-4.3,llm_error,categorical_eligibility,False,The model returned zero without applying categorical eligibility through TANF non-cash status or the income and asset tests. Those tests are satisfied and produce a positive monthly allotment throughout the year.
+us,scenario_100,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used wages alone, an incorrect $198 standard deduction, and an excess-medical deduction while omitting the $453.64 excess-shelter deduction. It therefore did not derive the $237.09 monthly net income used for the benefit."
+us,scenario_100,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced net income to zero through medical deductions and paid a flat $766 maximum. The correct net income is $237.09, and the applicable maximums are $785 and later $802.60."
+us,scenario_100,snap,grok-build-0.1,llm_error,period_annualization,False,"The model returned a single estimated monthly maximum as the annual output. SNAP must be calculated for every month and summed across twelve months, with the $71.10 contribution deducted each month."
+us,scenario_100,snap,inkling,llm_error,taxable_income_or_deductions,False,The model used wages alone and a disabled-member medical deduction to estimate only $50 of net income. The traced gross income and deductions instead produce $237.09 of net income and a $71.10 contribution.
+us,scenario_100,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so the requested output was missing."
+us,scenario_100,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted premiums and medical expenses until net income was zero and then paid $785 for every month. The traced deductions leave $237.09 net income, and later months use a $802.60 maximum."
+us,scenario_100,snap,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model stopped after the 20% earned-income deduction, omitting the $209 standard deduction and $453.64 excess-shelter deduction, then subtracted roughly $389 rather than 30% of net income from the maximum. The correct household contribution is only $71.10 per month."
+us,scenario_100,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used wages alone and an erroneous $182 medical deduction, producing net income near zero. It omitted the additional countable income and the $453.64 excess-shelter deduction that together yield $237.09 net income."
+us,scenario_100,snap,qwen-3.7-max,llm_error,period_annualization,False,"The model double-counted the $710 overtime premium by adding it to annual gross wages even though wages already include overtime. It also claimed a $766 monthly maximum but annualized the result to $12,444, which exceeds twelve times that stated maximum."
+us,scenario_100,snap,qwen3.8-max,llm_error,asset_resource,False,"The model incorrectly disqualified the household on resources. Its $2,800 of countable assets is within the applicable limit, and categorical eligibility through TANF non-cash status is satisfied."
+us,scenario_100,ssi,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the head's generic disability fact as satisfying SSI's categorical disability requirement and proceeded directly to the earned-income calculation. PolicyEngine's SSI-specific aged/blind/disabled indicator is false for the head, so no SSI benefit-rate or income-exclusion calculation applies; its submitted value also contradicts its own stated $9,480 calculation."
+us,scenario_100,ssi,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model equated the head's generic disability status with SSI categorical eligibility. The head's SSI-specific aged/blind/disabled indicator is false, so the earned-income reduction it applied is irrelevant and SSI is zero."
+us,scenario_100,ssi,gpt-5.5,llm_error,categorical_eligibility,False,"The model assumed the head was an SSI-qualified disabled adult and subtracted countable earnings from a federal benefit maximum. The head fails SSI's aged, blind, or qualifying-disabled criterion, so the calculation stops before income exclusions or the federal benefit rate are applied."
+us,scenario_100,ssi,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model declared the head eligible solely from the generic disability fact and calculated a reduced federal SSI benefit. PolicyEngine's SSI-specific categorical eligibility indicator is false for the head, making the benefit zero before any earned-income calculation."
+us,scenario_100,ssi,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated the head as an SSI-qualified disabled person and reduced the individual maximum by countable earnings. The head does not satisfy SSI's aged, blind, or qualifying-disabled condition, so no maximum-benefit calculation applies."
+us,scenario_100,ssi,grok-build-0.1,llm_error,categorical_eligibility,False,"The model converted the generic disability input directly into SSI categorical eligibility, then applied the $20 and $65 exclusions and the earned-income disregard. Because the head's SSI-specific aged/blind/disabled indicator is false, those income rules are never reached."
+us,scenario_100,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SSI value or explanation, violating the required structured-output contract."
+us,scenario_100,ssi,ox-alpha,llm_error,categorical_eligibility,False,"The model assumed the 46-year-old head's generic disability status met SSI's qualifying-disability test and then calculated countable wages. The SSI-specific categorical indicator is false, so the head is ineligible regardless of the wage calculation; neither child qualifies either."
+us,scenario_100,ssi,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model treated the generic disability input as SSI categorical eligibility and therefore performed an income-reduction calculation that should never occur. It also added the separately listed overtime premium to annual gross wages and invented an IRWE adjustment despite instructions not to infer unlisted expenses, but categorical ineligibility alone makes SSI zero."
+us,scenario_100,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model applied a 10% Montana EITC rate instead of 20% and improperly reduced federal EITC earned income by desired traditional retirement contributions. The federal EITC base is $2,365.89, producing $473.18 at the Montana rate."
+us,scenario_100,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly asserted that Montana has no applicable refundable EITC. This household qualifies for the refundable credit equal to 20% of its $2,365.89 federal EITC."
+us,scenario_100,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model correctly identified the $2,366 federal EITC but applied a 10% Montana rate instead of 20%, then returned $297 despite its own $237 computation. The correct state credit is $473.18."
+us,scenario_100,state_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model recognized a positive federal and Montana EITC but applied the wrong 10% rate and then discarded the credit entirely. Montana refunds 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,The model incorrectly treated Montana's EITC as nonrefundable and limited it by zero tax liability. The 2026 credit is refundable and equals 20% of the federal EITC.
+us,scenario_100,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model correctly calculated federal EITC of about $2,366 but used a 10% Montana percentage instead of 20%. That rate error halves the correct $473.18 credit."
+us,scenario_100,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model incorrectly classified the Montana EITC as nonrefundable. It is refundable in 2026 and pays 20% of this household's federal EITC.
+us,scenario_100,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used both the wrong 10% Montana rate and an unexplained federal EITC base of $371. Two qualifying children and $5,915 of earnings yield a $2,365.89 federal EITC, and Montana refunds 20%."
+us,scenario_100,state_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model correctly derived the federal EITC near $2,366 but applied a 10% Montana rate. The applicable rate is 20%, yielding $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted Montana's refundable EITC. This low-income worker with two qualifying children receives 20% of the federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model wrongly stated that Montana lacks a refundable EITC and also treated the household as having zero earned income despite $5,915 of wages. Those wages generate a $2,365.89 federal EITC and a $473.18 Montana credit."
+us,scenario_100,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model incorrectly set federal EITC to zero. The head has $5,915 of earned income and two qualifying children, producing $2,365.89 of federal EITC and $473.18 of Montana EITC."
+us,scenario_100,state_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,The model applied a 3% Montana EITC rate instead of 20% and failed to carry even its stated positive calculation into the submitted zero. The correct refundable amount is $473.18.
+us,scenario_100,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model falsely stated that Montana has no individual income tax. Montana has an income tax and a refundable EITC equal to 20% of the federal credit, producing $473.18 here."
+us,scenario_100,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model omitted the applicable Montana refundable EITC. The household's wages and two qualifying children generate a $2,365.89 federal EITC, of which Montana refunds 20%."
+us,scenario_100,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model incorrectly concluded that the benchmark produces no refundable Montana credit for this profile. The profile qualifies for a $473.18 Montana EITC.
+us,scenario_100,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used the wrong 10% Montana rate and overstated federal EITC as $4,207. At the 40% phase-in rate, $5,915 of earnings produces $2,365.89 federally, and Montana pays 20% of that amount."
+us,scenario_100,state_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly estimated the federal EITC but applied a 10% Montana percentage rather than 20%. The correct state credit is therefore $473.18, twice its answer."
+us,scenario_100,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model invented a $1,200 Montana young-child refundable credit and applied only a 10% EITC rate. The applicable refundable component is solely the Montana EITC at 20% of $2,365.89."
+us,scenario_100,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model added a nonexistent $1,200 young-child credit and halved the Montana EITC by using 10% instead of 20%. Only the $473.18 Montana EITC applies."
+us,scenario_100,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model omitted Montana's refundable EITC. This household qualifies for $473.18 based on 20% of its federal EITC.
+us,scenario_100,state_refundable_credits,grok-4.5,llm_error,categorical_eligibility,False,The model incorrectly stated that Montana offers no state EITC. Montana's refundable EITC is 20% of the federal credit and equals $473.18 here.
+us,scenario_100,state_refundable_credits,grok-4.6,llm_error,categorical_eligibility,False,The model incorrectly denied the existence of Montana's state EITC and focused only on property-tax credits. The refundable EITC alone supplies $473.18.
+us,scenario_100,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model failed to apply Montana's refundable EITC. The household qualifies for $473.18 through the 20%-of-federal-credit calculation.
+us,scenario_100,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model invented a $1,200 refundable Montana child credit, used the wrong 10% EITC rate, and understated the federal EITC as about $750. The sole applicable credit is 20% of the $2,365.89 federal EITC, or $473.18."
+us,scenario_100,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,The model incorrectly stated that Montana has no refundable state EITC and treated property or rent facts as necessary. The earnings-and-children pathway independently produces a $473.18 refundable Montana EITC.
+us,scenario_100,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,The model correctly calculated the federal EITC but used a 10% Montana rate instead of 20%. The correct refundable credit is $473.18.
+us,scenario_100,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model falsely stated that Montana has no state income tax. Montana imposes an individual income tax and provides a refundable EITC equal to 20% of the federal credit.
+us,scenario_100,state_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,The model used the correct federal EITC base but applied a 10% Montana rate instead of 20%. This produced exactly half of the correct $473.18 amount.
+us,scenario_100,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,The model incorrectly denied Montana's earned income tax credit and treated zero tax liability as eliminating refundable credits. The refundable Montana EITC applies regardless of pre-credit liability and equals $473.18.
+us,scenario_100,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model omitted the Montana refundable EITC. Applying the 20% state percentage to the $2,365.89 federal EITC yields $473.18."
+us,scenario_100,tanf,claude-fable-5,llm_error,thresholds_rates,False,"The model first calculated positive eligibility and a positive monthly grant, then contradicted that calculation by declaring the earnings above Montana's standards. Applying the 2026 Montana schedule to the stated earnings yields $6,064.96 rather than zero."
+us,scenario_100,tanf,claude-haiku-4.5,llm_error,asset_resource,False,"The model used an incorrect $2,000 resource limit to disqualify the household and also mislabeled the three-person household as a household of four. Montana's modeled resource and income tests do not disqualify this assistance unit."
+us,scenario_100,tanf,claude-opus-4.7,llm_error,thresholds_rates,False,"The model asserted that countable earnings eliminated the grant without applying Montana's 2026 benefit formula, even though it recognized that earnings were low and take-up was assumed. The applicable calculation produces $6,064.96."
+us,scenario_100,tanf,claude-opus-4.8,llm_error,thresholds_rates,False,"The model approximated the grant using a guessed monthly maximum of about $595 and failed to apply the exact 2026 Montana payment schedule and income adjustment. That substitution overstated the annual benefit as $7,140 instead of $6,064.96."
+us,scenario_100,tanf,claude-opus-5,llm_error,categorical_eligibility,False,"The model declared that PolicyEngine generally pays no TANF without identifying any failed eligibility test. This low-income parent-and-two-child assistance unit passes Montana's modeled eligibility rules and receives $6,064.96."
+us,scenario_100,tanf,claude-sonnet-4.6,llm_error,asset_resource,False,"The model imposed an outdated approximate $1,000 liquid-asset ceiling and used it to deny assistance. Under Montana's 2026 modeled resource rules, the $2,800 bank balance and excluded low-value vehicle do not eliminate TANF eligibility."
+us,scenario_100,tanf,claude-sonnet-5,llm_error,thresholds_rates,False,"The model substituted an obsolete or unrelated $327 monthly maximum and an estimated disregard formula for Montana's 2026 schedule. The correct state formula yields $6,064.96 annually."
+us,scenario_100,tanf,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a guessed $605 monthly maximum and a $200-plus-50-percent earnings disregard. Those are not the operative 2026 Montana parameters in the benchmark, so its $5,502 calculation understates the grant."
+us,scenario_100,tanf,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $588 payment standard and treated 75 percent of earnings above $200 as countable, then even rounded $4,416 to $4,419 without a valid step. Montana's exact 2026 formula yields $6,064.96."
+us,scenario_100,tanf,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model asserted failure of unspecified state requirements despite the parent, two dependent children, low income, and qualifying resources. Montana's modeled eligibility pathway produces a positive $6,064.96 grant."
+us,scenario_100,tanf,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model treated SSI receipt as eliminating TANF without applying the assistance-unit and income rules. The household remains eligible under the modeled Montana calculation and receives $6,064.96."
+us,scenario_100,tanf,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a $200-plus-50-percent earnings disregard and an approximate payment amount instead of Montana's exact 2026 parameters. That incorrect schedule produced $5,298 rather than $6,064.96."
+us,scenario_100,tanf,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model reduced the grant using an unspecified estimated earned-income disregard rather than the exact Montana 2026 benefit computation. The operative calculation produces $6,064.96, not $5,298.48."
+us,scenario_100,tanf,gemini-3.7-flash,llm_error,other,False,"The submitted numeric answer was zero while its explanation ended with value = 4824 and stated that the household qualified. This violated the required agreement between the numeric value and explanation, and neither figure equals the $6,064.96 calculation."
+us,scenario_100,tanf,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the separately listed FLSA overtime premium to annual gross wages even though the prompt says gross wages already include overtime, then asserted that earnings exceeded the payment standard. It also invoked work and resource failures that do not apply, causing an erroneous zero benefit."
+us,scenario_100,tanf,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated the absence of an explicit indication that TANF was payable as grounds for zero. The prompt assumes program take-up, and Montana's eligibility computation awards $6,064.96."
+us,scenario_100,tanf,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model required an extra TANF trigger even though the household facts establish a low-income parent with two dependent children and the prompt assumes take-up. Applying Montana's rules produces $6,064.96."
+us,scenario_100,tanf,gpt-5.5,llm_error,thresholds_rates,False,"The model estimated a $588 monthly standard and about $150 of countable earnings rather than applying the exact 2026 Montana parameters. Its resulting $438 monthly grant understates the annual benefit of $6,064.96."
+us,scenario_100,tanf,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model replaced Montana's exact payment schedule and earnings treatment with an unexplained estimate of $359.50 per month. The state computation yields $6,064.96 annually."
+us,scenario_100,tanf,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model used the wrong earned-income disregard and family-of-three payment standard, producing the same approximate $4,419.75 shortcut as a $588-minus-$219.6875 calculation. Montana's 2026 formula yields $6,064.96."
+us,scenario_100,tanf,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model treated SSI and wages as eliminating the cash grant without applying Montana's assistance-unit and countable-income rules. The modeled interaction leaves a positive TANF benefit of $6,064.96."
+us,scenario_100,tanf,grok-4.3,llm_error,categorical_eligibility,False,"The model defaulted to zero because it did not calculate TANF from the supplied household facts. With assumed take-up, Montana's eligibility and payment rules produce $6,064.96."
+us,scenario_100,tanf,grok-4.5,llm_error,thresholds_rates,False,"The model used a $588 monthly standard and a $200-plus-25-percent disregard formula that are not the benchmark's operative 2026 Montana parameters. That shortcut produced about $4,420 instead of $6,064.96."
+us,scenario_100,tanf,grok-4.6,llm_error,thresholds_rates,False,"The model subtracted approximately $220 of countable earnings from an incorrect $588 monthly payment standard. Applying Montana's exact 2026 schedule yields $6,064.96 rather than $4,416."
+us,scenario_100,tanf,grok-build-0.1,llm_error,thresholds_rates,False,"The model combined a guessed $675 maximum grant with an incorrect $200-plus-25-percent disregard. Those parameter errors yield $5,464 instead of the modeled $6,064.96."
+us,scenario_100,tanf,inkling,llm_error,thresholds_rates,False,"The model relied on an approximate $588 maximum and a $200-plus-25-percent earnings disregard. Montana's exact 2026 benefit schedule produces $6,064.96, not the resulting $4,420 estimate."
 us,scenario_100,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no TANF output or explanation. It therefore failed the required structured-output contract before any substantive TANF calculation could be evaluated.
-us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"The model correctly excluded the vehicle and treated the bank balance as within its assumed resource limit, but then used a $588 grant standard and counted 75% of earnings above $200. That parameter formula is inapplicable to the 2026 Montana calculation yielding $6,064.96."
-us,scenario_100,tanf,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the family qualifies and that the benefit should be nonzero, then set it to zero based on generic time-limit and work-requirement concerns unsupported by the household facts. It failed to apply the benchmark’s assumed take-up and constant-year facts to calculate the positive grant."
-us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted the $710 overtime premium by adding it to gross wages, contrary to the instruction that gross wages already include overtime. It then compared this inflated income to a guessed annual threshold without applying Montana’s benefit formula and incorrectly reduced the award to zero."
-us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly classified $5,915 of annual wages for a three-person family as too high for Montana TANF. The applicable 2026 income limits and payment calculation leave the household eligible for $6,064.96."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's stated bracket calculation produced $13,058, yet it submitted $12,064 without any supporting computation. It also incorrectly treated personal auto-loan interest as an additional deduction alongside the standard deduction."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used a $14,600 standard deduction instead of $16,100, included nontaxable veterans benefits in gross income, and applied internally inconsistent brackets. Its stated arithmetic does not produce its submitted $11,435."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model correctly derived $85,030 of taxable income and explicitly calculated $13,418.60, then submitted $14,716. The final value contradicts its complete and correct tax computation."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly derived taxable income of $85,030 and tax of approximately $13,419, then submitted $13,876 without a corresponding adjustment. The submitted value is inconsistent with its own calculation."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an estimated $15,750 standard deduction instead of $16,100 and estimated bracket thresholds rather than the applicable 2026 parameters. This raised taxable income from $85,030 to $85,380 and produced the wrong tax."
-us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly calculated AGI of $101,130 and taxable income of $85,030, but used incorrect bracket thresholds and then submitted $13,322 even though its own displayed bracket arithmetic totaled $13,523.60. The final answer follows neither the applicable brackets nor its stated calculation."
-us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted $3,840 of listed health premiums from wages as though they were pre-tax payroll deductions, although the facts provide expenses rather than a pre-tax wage exclusion. It also used a $15,400 standard deduction instead of $16,100, understating AGI and taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and applied restored personal exemptions, a smaller standard deduction, and 10%/15%/25% brackets. The applicable computation uses the $16,100 standard deduction, no personal exemption, and the 10%/12%/22% brackets at this income."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model ignored the $1,870 deductible capital loss and used a $15,300 standard deduction, producing $87,700 rather than $85,030 of taxable income. Tax must be computed after both the capital-loss adjustment and the $16,100 standard deduction."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified AGI of $101,130 but replaced the actual deduction-and-bracket computation with an unsupported estimate of $15,400. Applying the $16,100 standard deduction and the applicable 2026 brackets yields $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with a personal exemption, an approximately $8,250 standard deduction, and 10%/15%/25% rates. The applicable 2026 rules use a $16,100 standard deduction, no personal exemption, and 10%/12%/22% rates through this taxable-income range."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly reverted to pre-TCJA deductions and brackets, subtracting a personal exemption and applying 15% and 25% marginal rates. The correct taxable income is $85,030 and remains in the 22% bracket."
-us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used an $8,300 standard deduction plus a restored $5,300 personal exemption instead of the $16,100 standard deduction with no personal exemption. It consequently applied the wrong 2026 bracket regime to overstated taxable income."
-us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $15,450 standard deduction and estimated bracket thresholds rather than the applicable $16,100 deduction and 2026 brackets. This overstated taxable income by $650 and produced $13,611.04 instead of $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the explicitly tax-exempt private pension as taxable and failed to present the required AGI, standard-deduction, and bracket calculation. The pension and veterans benefits are excluded, leaving AGI of $101,130 and taxable income of $85,030."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized medical effects even though the standard deduction is $16,100 and exceeds itemized deductions of $1,871.33. Its $9,587 answer omits the valid derivation from $101,130 of AGI to $85,030 of taxable income and $13,418.60 of tax."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly used scheduled post-TCJA-expiration parameters, including a personal exemption and smaller standard deduction, to obtain about $87,380 of taxable income. The applicable rules provide a $16,100 standard deduction and no personal exemption, leaving $85,030 taxable."
-us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model claimed a phased-down qualified auto-loan-interest deduction that is not part of the reference taxable-income computation. Taxable income is $85,030 after the capital loss and $16,100 standard deduction, without an additional auto-loan-interest deduction."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an estimated $15,300 standard deduction rather than $16,100 and estimated bracket parameters. That overstated taxable income by $800 and produced an excessive liability."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA sunset, using an approximately $8,400 standard deduction, a personal exemption, and 10%/15%/25% brackets. The applicable calculation uses the $16,100 standard deduction, no personal exemption, and 10%/12%/22% brackets."
-us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly applied reverted pre-TCJA rules, including an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% marginal brackets. The correct 2026 regime produces $85,030 of taxable income taxed at 10%, 12%, and 22% through this range."
-us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly derived AGI of $101,130, the $16,100 standard deduction, and taxable income of $85,030, but used imprecise bracket thresholds. Applying the exact 2026 bracket parameters yields $13,418.60 rather than $13,426."
-us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. This is a missing-output failure rather than a substantive tax calculation.
-us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented a $1,174 phased-down auto-loan-interest deduction and subtracted it in addition to the standard deduction. The correct taxable income is $85,030 with no separate auto-loan-interest deduction, producing $13,418.60."
-us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model set pre-refundable-credit income tax to zero by invoking CTC, EITC, and refundable mechanisms despite there being no qualifying child and despite refundable credits being excluded from this output. With no applicable nonrefundable credits, the full bracket tax of $13,418.60 remains."
-us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used an estimated $15,750 standard deduction and incorrect bracket thresholds, but even those assumptions produced only $13,641 in its displayed calculation. It then submitted $14,827 without any additional tax or credit computation supporting the increase."
-us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly subtracted $4,824 of employee payroll tax as a nonrefundable federal income-tax credit; employee payroll tax is a separate liability and does not offset this output. It also failed to deduct the $1,870 capital loss against wages and used the wrong standard deduction and brackets."
+us,scenario_100,tanf,kimi-k3,llm_error,thresholds_rates,False,"The model correctly excluded the vehicle and accepted the bank balance but then used an incorrect $588 grant standard and 75-percent-of-earnings-above-$200 formula. The exact 2026 Montana computation yields $6,064.96 rather than $4,419.75."
+us,scenario_100,tanf,minimax-m3,llm_error,categorical_eligibility,False,"The model acknowledged that the family qualified for a positive benefit, then arbitrarily reduced it to zero based on generalized time-limit and work-requirement concerns not established by the facts. Montana's modeled rules award $6,064.96."
+us,scenario_100,tanf,ox-alpha,llm_error,taxable_income_or_deductions,False,The model invented approximately $780 per month of SSI as household income even though no SSI receipt amount was listed and unlisted numeric inputs must be zero. Adding that unsupported income caused the erroneous conclusion that countable income exceeded the TANF standard.
+us,scenario_100,tanf,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,The model double-counted the $710 overtime premium by adding it to gross wages despite the instruction that annual gross wages already include overtime. It then compared gross annual income to an estimated annual limit instead of applying Montana's exact countable-income and benefit formula.
+us,scenario_100,tanf,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly declared $5,915 of annual wages too high without applying Montana's income disregards and three-person benefit schedule. The household passes the modeled income test and receives $6,064.96."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model improperly deducted personal auto-loan interest from taxable income. It then submitted $12,064 even though its own alternative calculations produced $13,111 and $13,058, so the final value is unsupported by its reasoning."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $700 of veterans benefits in gross income and used a $14,600 standard deduction instead of $16,100. It also misapplied the brackets by taxing almost all taxable income at 12% and submitted a value that does not equal its stated bracket calculation."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly derived the exact $85,030 taxable income and $13,418.60 tax, then submitted $14,716. The failure is a final-answer transcription or consistency error, not a tax-rule error."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated income, deduction, bracket thresholds, and arithmetic produce approximately $13,419. It nevertheless submitted $13,876, an unsupported final-answer substitution."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated 2026 parameters, ultimately choosing a $15,750 standard deduction and invented bracket cutoffs instead of the $16,100 deduction and applicable thresholds. That left taxable income $350 too high and applied the wrong bracket segmentation."
+us,scenario_101,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly reached $85,030 of taxable income, but used inaccurate estimated bracket thresholds and calculated $13,523.60. It then submitted $13,322 without any arithmetic that produces that amount."
+us,scenario_101,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model invented a $3,840 above-the-line deduction for the listed health premiums, even though the facts do not establish pre-tax payroll treatment and the trace includes no such adjustment. It also used a $15,400 standard deduction rather than $16,100, understating taxable income and tax."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA individual provisions expired for 2026 and applied a reduced standard deduction, a personal exemption, and reverted 10%/15%/25% rates. It also invented pre-tax treatment for the $1,920 premium; the applicable computation uses the $16,100 standard deduction and 10%/12%/22% brackets."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model ignored the $1,870 capital-loss deduction and used a $15,300 standard deduction, producing $87,700 rather than $85,030 of taxable income. Its resulting bracket tax therefore rests on the wrong tax base."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly identified AGI of $101,130 but replaced the required deduction-and-bracket computation with an unsupported rough estimate of $15,400. Applying the $16,100 standard deduction and the applicable brackets to $85,030 yields $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration regime with a personal exemption and 10%/15%/25% brackets. The 2026 computation instead uses a $16,100 standard deduction, no personal exemption, and the applicable 10%/12%/22% brackets."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly treated 2026 as governed by reverted pre-TCJA rules, using an $8,300 standard deduction, a $5,300 personal exemption, and 15% and 25% marginal brackets. Those parameters overstated the tax relative to the applicable $16,100 standard deduction and 10%/12%/22% schedule."
+us,scenario_101,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used the wrong 2026 tax regime by substituting an $8,300 standard deduction plus a $5,300 personal exemption for the $16,100 standard deduction. It consequently taxed $87,530 rather than $85,030 under incorrect bracket parameters."
+us,scenario_101,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used estimated parameters: a $15,450 standard deduction and approximate bracket thresholds. The applicable $16,100 deduction produces $85,030 of taxable income, and the correct bracket calculation produces $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated some portion of the explicitly tax-exempt pension as taxable and described the capital loss as a small taxable gain rather than a $1,870 deduction. Those income-definition errors prevented it from deriving the $101,130 AGI and $85,030 taxable income used for the tax calculation."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified medical itemized effects even though the $16,100 standard deduction exceeds the $1,871.33 itemized deductions. Its $9,587 answer implies substantial deductions or nonrefundable credits that do not exist for this filer."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly included a personal exemption in its scheduled 2026 computation and arrived at about $87,380 of taxable income. The applicable calculation has no personal exemption and uses the $16,100 standard deduction, leaving $85,030 taxed under the 2026 10%/12%/22% brackets."
+us,scenario_101,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model improperly claimed a phased-down qualified auto-loan-interest deduction. No such deduction enters the trace, so taxable income remains $85,030 after only the capital-loss adjustment and $16,100 standard deduction."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model used an estimated $15,300 standard deduction instead of $16,100 and unspecified projected bracket thresholds. This overstated taxable income by $800 and produced the wrong bracket tax."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly assumed a post-TCJA-sunset regime with an approximately $8,400 standard deduction, a personal exemption, and 10%/15%/25% brackets. The applicable 2026 calculation uses a $16,100 standard deduction, no personal exemption, and 10%/12%/22% rates at this income."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used a standard deduction near $15,000 rather than $16,100 and incorrect bracket cutoffs. It therefore taxed $86,130 instead of $85,030 and overstated the liability."
+us,scenario_101,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model applied reverted TCJA rules, including an $8,300 standard deduction, a $5,300 personal exemption, and 15%/25% brackets. The applicable rules instead produce $85,030 of taxable income under the 10%/12%/22% schedule."
+us,scenario_101,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model correctly derived AGI of $101,130, the $16,100 standard deduction, and taxable income of $85,030. Its $13,426 result reflects a small bracket-threshold or arithmetic error because the applicable bracket calculation on that base is $13,418.60."
+us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. This is a missing-output contract failure.
+us,scenario_101,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $1,174 of auto-loan interest after inventing an AGI phase-down formula. The auto-loan interest does not reduce taxable income here, which remains $85,030 rather than $83,856."
+us,scenario_101,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model set the tax to zero despite acknowledging that wages and the standard deduction leave positive tax. It also invoked CTC and EITC even though there are no qualifying children, and refundable mechanisms do not reduce this before-refundable-credits output."
+us,scenario_101,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model used an estimated $15,750 standard deduction and inaccurate bracket thresholds, calculating a preliminary tax of $13,641. It then submitted $14,827 without identifying any credit, tax, or arithmetic step that converts its calculation to that value."
+us,scenario_101,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,payroll_tax_base,False,"The model wrongly treated employee payroll tax as a nonrefundable federal income-tax credit and subtracted $4,824 from income tax. It also failed to deduct the $1,870 capital loss from wages and used an incorrect standard deduction and bracket calculation; payroll tax is a separate output and does not offset this liability."
 us,scenario_101,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_101,payroll_tax,claude-haiku-4.5,llm_error,other,False,"The model correctly derived $6,386 of Social Security tax and $1,493.50 of Medicare tax, then submitted $8,149.50 despite its own arithmetic totaling $7,879.50. Its unexplained $270 addition is not produced by a wage-base cap, rounding, Additional Medicare Tax, or Texas employee payroll tax."
 us,scenario_101,payroll_tax,claude-sonnet-5,llm_error,other,False,"The model correctly calculated total employee payroll tax as $7,879.50 but submitted $7,854.00. The $25.50 reduction has no basis in any payroll-tax component stated in its reasoning."
@@ -6038,32 +6340,33 @@ us,scenario_101,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,Fals
 us,scenario_101,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated ordinary withholding of employee FICA taxes as though it eliminated annual payroll-tax liability. W-2 wages of $103,000 generate $6,386 of employee Social Security tax and $1,493.50 of employee Medicare tax regardless of employer-sponsored insurance or Texas's lack of a state payroll tax."
 us,scenario_101,payroll_tax,qwen3.8-max,llm_error,other,False,"The model correctly summed Social Security and Medicare taxes to $7,879.50, then added an unexplained $152. No Additional Medicare Tax or mandatory Texas employee payroll tax applies, so that extra amount has no payroll-tax source."
 us,scenario_101,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,"The model invented a Texas individual income tax liability by applying an unstated tax rate to taxable income. Texas imposes no individual state income tax, so the correct state income tax before refundable credits is $0, not $6,035."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own no-premium-exclusion calculation produces about $6,092, but it submitted $5,459 without a computation supporting that amount. It also vacillated over subtracting the $7,989 ESI premium even though the listed wages enter gross income as stated."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model incorrectly subtracted estimated federal withholding from income-tax liability. Withholding is a payment toward tax and does not reduce federal_income_tax_before_refundable_credits; the liability is computed from $55,000.80 of taxable income."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model used an understated estimated standard deduction and computed tax near $6,204, then submitted $5,764 with no corresponding deduction, credit, or arithmetic. The correct deductions total $32,200.20 and leave $55,000.80 taxable."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model’s stated bracket calculation yields about $6,092, yet it submitted $6,633. No tax bracket, surtax, or other adjustment in its reasoning generates the extra $541."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model essentially reconstructed the correct taxable income and explicitly calculated tax of about $6,104, but then submitted $5,983. That final value contradicts its own bracket computation."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,800 standard deduction instead of the 2026 $32,200 deduction and omitted the $0.20 QBI deduction. This overstated taxable income by $1,400.20 and therefore overstated tax."
-us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model’s stated taxable-income and bracket arithmetic produces about $6,056, but it submitted $6,800. The $1 of self-employment income cannot create the unexplained $744 increase."
-us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied a $30,000 standard deduction instead of $32,200 and omitted the $0.20 QBI deduction. It consequently taxed $57,201 rather than $55,000.80."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly used a post-sunset regime with a $16,000 standard deduction, personal exemptions, and a 15% bracket. The applicable 2026 computation instead uses the $32,200 joint standard deduction and the 10% and 12% rate schedule at this taxable-income level."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model claimed to apply the brackets to $57,201 of taxable income but submitted $4,124, which does not follow from that base under the 2026 joint schedule. The correct deduction calculation produces $55,000.80 taxable and $6,104.10 of tax."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset, introduced personal exemptions, and used projected pre-TCJA brackets. It also subtracted the separately listed $7,989 ESI premium from wages despite the gross wage amounts being the income inputs."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied an inapplicable post-TCJA-expiration $16,000 standard deduction and 15% bracket. It also reduced the stated wages by the $7,989 ESI premium, rather than beginning with the trace’s $87,201 gross income."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,989 ESI premium from the stated gross wages and used a $30,800 standard deduction. Gross income remains $87,201, and deductions are $32,200.20, producing $55,000.80 taxable."
-us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer invokes personal exemptions under a supposed 2026 regime and returns $7,915.15 without the applicable computation. The correct 2026 joint standard deduction and rate schedule applied to $55,000.80 yield $6,104.10."
-us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model’s own arithmetic calculates $6,398.12 and states that no nonrefundable credits apply, but it submitted $1,737. The submitted amount has no support in its reasoning and does not follow from the household’s taxable income."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model submitted $3,357 without identifying any deduction or credit that reduces the bracket tax to that amount. The $32,200.20 of deductions leave $55,000.80 taxable, and no nonrefundable credit applies."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model incorrectly reduced the liability to zero despite acknowledging there are no child, dependent, or other specified nonrefundable credits. The standard deduction does not eliminate the $55,000.80 of taxable income."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated wages by the $7,989 employer-sponsored insurance premium. The computation starts from $87,201 of gross income and subtracts only the $32,200 standard deduction and $0.20 QBI deduction here."
-us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model improperly treated the $7,989 ESI premium as an additional reduction from the listed gross wages. Using the stated $87,201 gross income and $32,200.20 of deductions produces $55,000.80 taxable."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used taxable income of roughly $57,200, reflecting an understated deduction near $30,000. The 2026 standard deduction plus QBI deduction totals $32,200.20 and reduces taxable income to $55,000.80."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,840 standard deduction rather than $32,200 and omitted the $0.20 QBI deduction. This overstated taxable income and tax."
-us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted the spouse’s $7,989 ESI premium from the stated wage income and also used a $30,800 standard deduction. The correct gross income is $87,201 and the correct deductions total $32,200.20."
-us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output. It therefore failed the required structured-output contract.
-us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model estimated the joint standard deduction at $30,750 instead of using $32,200 and omitted the $0.20 QBI deduction. It therefore taxed $56,451 rather than $55,000.80."
-us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a qualifying individual and applied a Child and Dependent Care Credit even though the household contains only the two spouses and lists no dependent-care expenses. Disability of a spouse does not by itself generate the asserted CDCC, and its submitted $3,602 also does not reconcile with its stated $6,324 less $600."
-us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model invented two qualifying children and subtracted a $4,000 Child Tax Credit even though the household lists no children. It also miscomputed the pre-credit bracket tax on its own stated taxable income; no CTC applies to the actual two-adult household."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own no-ESI calculation was approximately $6,092, but it submitted $5,459 without any computation producing that amount. It also used an estimated $32,300 standard deduction instead of the exact $32,200 deduction plus the $0.20 QBI deduction."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,other,False,"The model incorrectly treated estimated wage withholding as a subtraction from federal income tax before refundable credits. Withholding is a payment, not a nonrefundable credit, so the tax output remains the liability computed from taxable income rather than $5,620 minus $2,810."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model calculated approximately $6,204 from its estimated deduction and brackets, then submitted $5,764 with no corresponding adjustment or credit. The exact computation uses the $32,200 standard deduction, $0.20 QBI deduction, and 2026 brackets to reach $6,104.10."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model explicitly computed approximately $6,092 and then submitted $6,633 despite identifying no credit, surtax, or other modification. Its final value is disconnected from its stated tax calculation."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model correctly reconstructed approximately $6,104 using the applicable deduction and brackets, but submitted $5,983. No stated computation or nonrefundable credit reduces the tax to its submitted value."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model estimated the 2026 standard deduction as $30,800 instead of using $32,200 and omitted the $0.20 QBI deduction. That overstated taxable income as $56,401 rather than $55,000.80 and therefore overstated the tax."
+us,scenario_102,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model calculated approximately $6,056 from its estimated parameters and then submitted $6,800, attributing the $744 increase to a $1 self-employment item that has only a negligible effect. Its submitted value does not follow from its stated calculation."
+us,scenario_102,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used the $30,000 married-filing-jointly standard deduction from the wrong year instead of the 2026 amount of $32,200 and omitted the $0.20 QBI deduction. It consequently taxed $57,201 instead of $55,000.80."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model applied an expired-law framework with a $16,000 standard deduction, personal exemptions, and a 15% second bracket instead of the operative 2026 $32,200 standard deduction and 12% bracket. It also improperly reduced the stated wages by the separately listed $7,989 employer-sponsored insurance premium."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model claimed to apply 2026 brackets to $57,201 of taxable income but submitted $4,124, which is not the tax produced by those inputs. It also used a $30,000 standard deduction rather than $32,200 and omitted the $0.20 QBI deduction."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model incorrectly assumed a TCJA sunset and applied a reduced standard deduction, personal exemptions, and pre-TCJA rates. It also subtracted the $7,989 ESI premium from wages even though the listed wage amounts are the income inputs and the premium is not an authorized adjustment to them."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly used post-sunset parameters, including a $16,000 standard deduction and a 15% second bracket, rather than the operative 2026 schedule. It also double-counted the ESI exclusion by subtracting the separately listed $7,989 premium from the stated wages."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the separately listed $7,989 employer-sponsored insurance premium from the stated wage income. It also used a $30,800 standard deduction instead of $32,200 and omitted the $0.20 QBI deduction, producing $48,412 rather than $55,000.80 of taxable income."
+us,scenario_102,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted $7,915.15 reflects the wrong 2026 legal regime because the model invoked personal exemptions alongside the standard deduction. The operative calculation has no personal exemptions and taxes $55,000.80 under the 10% and 12% married-filing-jointly brackets."
+us,scenario_102,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model computed $6,398.12 from its stated assumptions and then submitted $1,737 without identifying any credit or adjustment that produces that reduction. It also used a $30,000 standard deduction instead of $32,200 and omitted the $0.20 QBI deduction."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model submitted $3,357 without showing a taxable-income or bracket calculation and correctly acknowledged that disability created no additional nonrefundable credit. The correct deductions leave $55,000.80 taxable, whose 2026 bracket tax is $6,104.10, so its number embodies an unsupported tax reduction."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model incorrectly reduced the liability to zero despite stating that no child, dependent, or other nonrefundable credit applied. The standard deduction does not eliminate the household’s $55,000.80 of taxable income, which generates $6,104.10 of tax."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the separately listed $7,989 employer-sponsored insurance premium from the stated wage amounts as a pre-tax adjustment. The wage inputs remain $87,200, and after the $32,200 standard deduction and $0.20 QBI deduction taxable income is $55,000.80."
+us,scenario_102,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model improperly reduced the stated wages by the $7,989 employer-sponsored insurance premium. That double-counted an ESI exclusion and understated the taxable income that remains after the $32,200 standard deduction and $0.20 QBI deduction."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated taxable income as approximately $57,200, which corresponds to using a roughly $30,000 standard deduction. The exact 2026 standard deduction is $32,200, and the additional $0.20 QBI deduction leaves $55,000.80 taxable."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,840 standard deduction rather than the exact 2026 amount of $32,200 and omitted the $0.20 QBI deduction. This overstated taxable income by $1,360.20 and produced excessive tax."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly applied a TCJA-expiration regime with a $16,550 standard deduction, personal exemptions, and a 15% second bracket. The operative 2026 computation uses a $32,200 standard deduction, no personal exemptions, and a 12% second bracket."
+us,scenario_102,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model improperly subtracted the $7,989 employer-sponsored insurance premium from the separately stated wages, understating AGI by that amount. It also used a $30,800 standard deduction rather than $32,200 and omitted the $0.20 QBI deduction."
+us,scenario_102,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, so the required numeric value could not be parsed."
+us,scenario_102,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an estimated $30,750 standard deduction instead of the exact $32,200 amount and omitted the $0.20 QBI deduction. It therefore taxed $56,451 rather than $55,000.80."
+us,scenario_102,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model invented a Child and Dependent Care Credit even though the household contains only the two spouses and no qualifying child or other dependent-care individual or listed care expense. It also claimed a reduction from about $6,324 to $3,602 that exceeds its stated $600 credit and is arithmetically inconsistent."
+us,scenario_102,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented two qualifying children and subtracted a $4,000 Child Tax Credit even though no children are listed. It also overstated pre-credit tax as $8,631; the applicable deductions and brackets produce $6,104.10 before refundable credits, with no CTC to subtract."
 us,scenario_102,federal_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a $3,995 refundable disability-related credit even after correctly ruling out EITC, refundable CTC, and the American Opportunity Credit. The Credit for the Elderly or the Disabled is nonrefundable, disability alone does not establish it for these working 26-year-olds, and no listed fact generates any refundable component."
 us,scenario_102,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for federal_refundable_credits, violating the required output contract."
 us,scenario_102,head_medicaid_eligible,claude-fable-5,llm_error,categorical_eligibility,False,"The model invented a medically needy spend-down pathway in the PolicyEngine determination and improperly treated rent as a Medicaid spend-down expense. The head qualifies for no Medicaid category, and the household's $87,201 MAGI exceeds every applicable income threshold, so the listed expenses and assets do not change the result."
@@ -6123,36 +6426,38 @@ us,scenario_104,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eli
 us,scenario_104,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age 72 plus Social Security below an approximate aged/disabled threshold as sufficient and categorically removed all $20,520 of veterans benefits from its income test. The applicable category determination yields NONE, with no SSI-based or other aged pathway establishing eligibility."
 us,scenario_104,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model correctly calculated net income of about $1,956 per month and explicitly found that it exceeded the elderly household's net-income limit, but then awarded $2,052 instead of enforcing the failed eligibility test. The minimum allotment applies only to an eligible one- or two-person household; it cannot replace a failed net-income test."
 us,scenario_104,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model did not carry the listed $8,389 annual employer-sponsored insurance premium through the elderly medical-expense deduction and then asserted an unsupported $196 monthly benefit. With all $34,616 of Social Security and veterans income counted, the allowable deductions still leave net income above the one-person limit, while the $540 monthly rent is below half of adjusted income and creates no excess-shelter deduction."
-us,scenario_104,state_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model wrongly concluded that $6,484 of annual rent was too low to generate the renter real property tax credit. Applying the senior-renter formula to the qualifying rent and household gross income produces the $375 refundable credit."
-us,scenario_104,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated earned-income and dependent-related credits as the complete set of applicable New York refundable credits. It omitted the real property tax credit available to this 72-year-old renter, which does not require earnings or dependents and equals $375."
-us,scenario_104,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,"The model incorrectly included $20,520 of veterans benefits in household gross income and therefore placed the household above the $18,000 real property tax credit ceiling. It then invented a $63 residual credit despite stating that no credit applied; excluding the veterans benefits and applying the renter formula yields $375."
-us,scenario_104,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model incorrectly made positive New York taxable or earned income a prerequisite for all refundable credits. The real property tax credit is available to the qualifying elderly renter even though Social Security and veterans benefits are excluded from New York taxable income, and it equals $375."
-us,scenario_104,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model incorrectly used zero AGI, no earnings, and no children to reject the real property tax credit. That credit follows a separate senior-renter pathway based on household gross income and rent, producing $375 here."
-us,scenario_104,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly counted the $20,520 veterans benefits in New York real property tax credit household gross income, producing $34,616 and a false failure of the $18,000 limit. Excluding those benefits leaves qualifying income below the ceiling, and the senior-renter calculation yields $375."
-us,scenario_104,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated the absence of taxable or earned income with the absence of refundable-credit eligibility. The elderly renter qualifies for New York’s real property tax credit under its separate rent-and-household-income rules, yielding $375."
-us,scenario_104,state_refundable_credits,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model stopped after rejecting the New York EITC for lack of earnings and omitted the real property tax credit. The head’s age, qualifying rent, and household gross income trigger a $375 refundable real property tax credit."
-us,scenario_104,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model incorrectly treated earnings and dependents as prerequisites for every New York refundable credit. The real property tax credit instead applies through the senior-renter pathway and equals $375.
-us,scenario_104,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The zero answer omitted the qualifying New York real property tax credit. A 72-year-old renter with the stated qualifying income and rent receives $375 under that credit’s formula.
-us,scenario_104,state_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model placed household gross income above $18,000 by counting the veterans benefits for the real property tax credit test. Those benefits are excluded from this computation, leaving qualifying income below the ceiling and producing a $375 credit."
-us,scenario_104,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model omitted the New York real property tax credit for an eligible elderly renter. Applying the household-income and rent formula yields $375.
-us,scenario_104,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly found that household gross income exceeded the real property tax credit limit. The credit calculation excludes the veterans benefits, so qualifying income remains below $18,000 and the renter receives $375."
-us,scenario_104,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model’s blanket finding of no refundable-credit eligibility omitted the senior-renter real property tax credit. The stated age, rent, and qualifying household income produce $375."
-us,scenario_104,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model falsely imposed a minimum-rent threshold that disqualified the household from the real property tax credit. The stated $6,484 annual rent is a valid basis for the senior-renter formula, which yields $375."
-us,scenario_104,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model overlooked the facts establishing eligibility for New York’s real property tax credit: age 72, renter status, qualifying rent, and qualifying household income. That credit contributes $375."
-us,scenario_104,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model focused on earned-income-related credits and omitted the refundable real property tax credit. The elderly renter pathway produces $375 without requiring earnings or dependents.
-us,scenario_104,state_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly asserted that average monthly rent above a renter limit makes the household ineligible. The stated rent is used in the New York real property tax credit formula for this eligible senior renter, producing $375."
-us,scenario_104,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model failed to recognize that the listed age and rent establish the relevant refundable-credit pathway. New York’s real property tax credit applies to this qualifying elderly renter and equals $375.
-us,scenario_104,state_refundable_credits,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model incorrectly treated earnings and children as necessary for any New York refundable credit. The real property tax credit uses the head’s senior status, rent, and qualifying household income and yields $375."
-us,scenario_104,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model wrongly rejected refundable credits solely because the elderly one-person household had no earnings or children. The separate senior-renter real property tax credit applies and produces $375.
-us,scenario_104,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model limited New York refundable credits to pathways requiring earnings or children. It omitted the real property tax credit for the qualifying 72-year-old renter, worth $375."
-us,scenario_104,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly counted veterans benefits toward the $18,000 real property tax credit household-income ceiling. Excluding those benefits leaves qualifying income under the limit, and the renter formula produces $375."
-us,scenario_104,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model considered the Empire State Child Credit but omitted New York’s real property tax credit. The latter does not require a child and awards this qualifying elderly renter $375.
-us,scenario_104,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model stopped after ruling out the EITC and child credit based on zero earnings and no dependents. It omitted the independent senior-renter real property tax credit pathway, which yields $375."
-us,scenario_104,state_refundable_credits,kimi-k2.6,llm_error,thresholds_rates,False,"The model incorrectly found that the real property tax credit income limit was exceeded by treating veterans benefits as household gross income. Those benefits are excluded from the relevant computation, leaving qualifying income under $18,000 and generating $375."
-us,scenario_104,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model calculated $34,616 of household gross income by adding veterans benefits to Social Security and therefore falsely failed the $18,000 renter-credit limit. Excluding the veterans benefits leaves $14,096 for this test, and the real property tax credit equals $375."
-us,scenario_104,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,"The model incorrectly assumed that no earnings or children meant no applicable state refundable credit. The elderly renter qualifies separately for New York’s real property tax credit, producing $375."
-us,scenario_104,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model incorrectly stated that the real property tax credit requires earned income or dependent children. It instead applies to this 72-year-old renter based on qualifying household income and rent, yielding $375."
-us,scenario_104,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly linked refundable-credit eligibility to New York tax liability, earnings, dependents, or an unspecified additional rent qualification. The stated rent and senior status satisfy the real property tax credit pathway, and its formula yields $375."
+us,scenario_104,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly concluded that the renter's $6,483.53 annual rent produced no real property tax credit. Applying New York's senior-renter real property tax credit formula yields $375."
+us,scenario_104,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model treated the absence of earnings and dependents as eliminating all refundable credits and omitted New York's real property tax credit. The 72-year-old renter receives $375 under that credit.
+us,scenario_104,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model incorrectly used $34,616 as disqualifying household gross income for the real property tax credit, then inserted an unsupported $63 amount. The applicable age-and-rent calculation produces a $375 refundable real property tax credit."
+us,scenario_104,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly treated zero New York taxable income as eliminating the real property tax credit. That refundable renter credit applies independently of positive taxable income and equals $375 here.
+us,scenario_104,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,The model used zero AGI and zero earnings as a blanket bar to refundable credits. It omitted the $375 New York real property tax credit generated by the taxpayer's senior status and rent.
+us,scenario_104,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model incorrectly combined Social Security and veterans benefits into a $34,616 disqualifying amount under its real property tax credit screen. PolicyEngine's applicable New York calculation awards this 72-year-old renter $375."
+us,scenario_104,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model focused on taxable income and earned-income credits and dismissed the real property tax credit without applying its renter formula. The senior renter's $6,483.53 rent produces a $375 refundable credit."
+us,scenario_104,state_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model assumed that no earned income meant no refundable New York credit. It omitted the real property tax credit, which does not require earned income and equals $375 here."
+us,scenario_104,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model treated earnings and dependents as the only bases for refundable state credits. It failed to apply New York's senior-renter real property tax credit, which yields $375."
+us,scenario_104,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,The model omitted the qualifying pathway for New York's real property tax credit. The taxpayer's age and rent generate a $375 refundable credit.
+us,scenario_104,state_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model incorrectly applied an $18,000 household-income exclusion to deny the real property tax credit. The applicable New York age-and-rent calculation yields $375."
+us,scenario_104,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,The model failed to recognize the New York real property tax credit for an elderly renter. Applying that credit produces $375.
+us,scenario_104,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model incorrectly screened the household out of the real property tax credit on gross income. PolicyEngine's New York renter-credit calculation based on age 72 and $6,483.53 of rent produces $375."
+us,scenario_104,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,The model asserted that no refundable credit applied without evaluating the senior-renter real property tax credit. That credit equals $375.
+us,scenario_104,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model incorrectly treated $6,484 of annual rent as below a required threshold for the real property tax credit. The New York formula applied to this senior renter's rent yields $375."
+us,scenario_104,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The model overlooked the facts establishing New York real property tax credit eligibility: age 72 and annual rent of $6,483.53. The resulting refundable credit is $375."
+us,scenario_104,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model considered earned-income-related credits but omitted New York's refundable real property tax credit. The senior renter receives $375 under that credit.
+us,scenario_104,state_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,The model incorrectly denied the real property tax credit because it viewed the average monthly rent as exceeding a renter limit. Applying the governing age-and-rent formula instead produces a $375 credit.
+us,scenario_104,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model failed to use the listed age and rent as the basis for New York's real property tax credit. Those facts produce a $375 refundable credit.
+us,scenario_104,state_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,The model treated earnings and children as necessary for every refundable New York credit. It omitted the senior-renter real property tax credit of $375.
+us,scenario_104,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,The model incorrectly assumed an elderly household without earnings or children had no refundable-credit pathway. New York's real property tax credit specifically applies to this renter and equals $375.
+us,scenario_104,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model limited refundable credits to income- or child-based programs. It omitted the $375 New York real property tax credit based on the taxpayer's age and rent.
+us,scenario_104,state_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly used a $34,616 household-income total to deny the real property tax credit under an $18,000 screen. The applicable senior-renter calculation awards $375."
+us,scenario_104,state_refundable_credits,grok-4.6,llm_error,state_local_rule,False,The model incorrectly screened the household out of IT-214 based on its income calculation. Applying the real property tax credit calculation to the 72-year-old renter yields $375.
+us,scenario_104,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,The model discussed the Empire State Child Credit but omitted the separate New York real property tax credit. The senior renter qualifies for a $375 refundable amount.
+us,scenario_104,state_refundable_credits,inkling,llm_error,state_local_rule,False,"The model considered only the EITC and Empire State Child Credit, making zero earnings and dependents dispositive. It omitted the $375 real property tax credit available to this elderly renter."
+us,scenario_104,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,The model incorrectly applied an income-limit exclusion to the New York real property tax credit. The applicable age-and-rent computation produces a $375 refundable credit.
+us,scenario_104,state_refundable_credits,kimi-k3,llm_error,state_local_rule,False,"The model incorrectly counted the full $34,616 of Social Security and veterans benefits against an $18,000 renter-credit limit and denied IT-214. PolicyEngine's applicable real property tax credit calculation yields $375."
+us,scenario_104,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model assumed that no earnings or children eliminated all state refundable credits. It omitted New York's $375 real property tax credit for the elderly renter.
+us,scenario_104,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model incorrectly treated approximately $34,616 of resources as disqualifying under the real property tax credit income test. The applicable renter-credit formula awards $375."
+us,scenario_104,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model incorrectly asserted that New York's real property tax credit requires earned income or dependent children. This senior renter qualifies through age and rent, producing a $375 credit."
+us,scenario_104,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model incorrectly tied refundable-credit eligibility to tax liability, earnings, dependents, or supposedly qualifying rent. New York's refundable real property tax credit applies to the listed senior renter and equals $375."
 us,scenario_107,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly included half of the $6,948 disability benefit in Social Security provisional income, inflating combined income from about $28,197 to $31,671 and taxable Social Security from $1,597.75 to $3,335.50. It also used an understated deduction total, leaving artificial taxable income instead of zero."
 us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model inflated taxable Social Security to about $3,336 by using the same erroneous $31,671 combined-income calculation that incorporates half of the disability benefit. Its approximately $15,450 deduction total was also below the applicable 2026 standard deduction plus age-65 addition, creating taxable income that does not exist."
 us,scenario_107,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $811.33 result is consistent with treating the disability benefits and excess Social Security as taxable while subtracting only the basic and age-65 standard deductions. The correct AGI is $15,439.75, which is already below those deductions and therefore produces no taxable income."
@@ -6176,17 +6481,18 @@ us,scenario_107,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The m
 us,scenario_107,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model improperly treated the earned-income, standard, shelter, and utility deductions as overcoming the household’s initial income ineligibility. It also contradicted its own formula: $292 minus 30% of $845 is about $38.50, not $292, so $3,504 is the maximum allotment rather than the result of its stated calculation."
 us,scenario_107,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model applied Ohio's 2.75% rate from the first dollar of its calculated $3,966 taxable income. That amount falls entirely within Ohio's zero-tax bracket, so tax before the senior credit is already $0 and cannot become $59.07."
 us,scenario_107,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_108,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $10,392 survivor-benefit amount as taxable non-Social-Security income, inflating AGI from $7,656 to $18,048. Its deduction estimate then left artificial taxable income; applying the deduction against the correct $7,656 AGI yields no taxable income and zero tax."
-us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model added the $10,392 survivor benefit to taxable wages and therefore started with an erroneous $18,048 AGI. Federal AGI is $7,656, which is fully absorbed by the applicable deduction and produces zero taxable income."
-us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model misclassified the $10,392 survivor benefit as a taxable pension and included it in its $18,048 AGI. The federal calculation includes only $7,656 of wages in AGI, so deductions reduce taxable income to zero; its post-TCJA deduction and personal-exemption calculation was also inapplicable."
-us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model wrongly included the $10,392 survivor benefit in AGI and computed tax from $18,048 rather than $7,656. It also assumed a TCJA-expiration deduction regime for 2026 instead of applying the operative parameters, but the correct AGI is already below the applicable deduction and yields zero taxable income."
-us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 survivor benefit as taxable income, overstating AGI by that full amount. With only $7,656 entering federal AGI, the applicable deduction eliminates taxable income and no 10% tax is generated."
-us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but incorrectly added the separate $10,392 survivor benefit to federal AGI. That produced its residual $348 of taxable income; AGI is instead $7,656 and the deduction reduces taxable income to zero."
-us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's $18,048 AGI wrongly includes the $10,392 survivor benefit as taxable income. Only the $7,656 of wages enters AGI, and the applicable deduction eliminates all taxable income; its personal-exemption framework was also inapplicable."
-us,scenario_108,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model labeled the $10,392 survivor benefit a taxable survivor pension and included it in AGI. This created the artificial $598 taxable-income remainder; the correct $7,656 AGI is fully offset by the applicable deduction."
-us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly treated the $10,392 survivor benefit as taxable and began from $18,048 of AGI instead of $7,656. It also applied a projected post-sunset personal-exemption and standard-deduction structure, whereas the operative calculation leaves the correct AGI with zero taxable income."
-us,scenario_108,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the $10,392 survivor benefit in federal AGI, creating $3,291 of artificial taxable income after its estimated deductions. The computation includes only $7,656 of wages in AGI, which is fully eliminated by the applicable deduction."
-us,scenario_108,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model wrongly added the separate $10,392 survivor benefit to wages, producing an erroneous $18,048 AGI and a $348 taxable-income remainder. Federal AGI is $7,656, so the applicable deduction reduces taxable income and tax to zero."
+us,scenario_108,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a taxable “non-SS survivor benefits” classification for the $10,392 and added it to wages, raising AGI from $7,656 to $18,048. Excluding that amount from federal AGI leaves the wages fully absorbed by the deduction and produces zero tax."
+us,scenario_108,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 survivor-benefit input as taxable income, yielding an erroneous $18,048 AGI. Federal AGI is $7,656, which is below the applicable deduction and leaves no taxable income."
+us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model relabeled the $10,392 survivor benefits as taxable pension benefits and included them in AGI. It also used an obsolete post-sunset deduction-and-personal-exemption construction, but excluding the survivor benefits alone reduces AGI to $7,656 and taxable income to zero."
+us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $10,392 survivor benefits to wages as taxable income and then assumed a TCJA-expiration deduction regime. The survivor benefits do not enter federal AGI, so the $7,656 AGI is fully offset and no tax remains."
+us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $10,392 survivor benefits as taxable pension income, producing $18,048 of AGI. With those benefits excluded, AGI is only $7,656 and the applicable deduction eliminates taxable income."
+us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although the model correctly excluded Social Security under the provisional-income test, it incorrectly included the separate $10,392 survivor benefits in AGI. Federal AGI is $7,656 rather than $18,048, so the deduction leaves zero taxable income rather than $348."
+us,scenario_108,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly counted the $10,392 survivor benefits in federal AGI and then applied an inapplicable personal-exemption framework. Excluding those benefits leaves $7,656 of AGI, entirely below the available deduction."
+us,scenario_108,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model called the $10,392 a survivor pension and included it in taxable income even though the input does not identify taxable pension income. Federal AGI is only $7,656, so no taxable income remains after the deduction."
+us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model misclassified the $10,392 survivor benefits as taxable and added them to wages, then used a post-TCJA-sunset personal-exemption calculation. The benefits do not enter federal AGI; the resulting $7,656 AGI is fully offset by the deduction."
+us,scenario_108,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly included the $10,392 survivor benefits in AGI and also substantially understated the deduction available to an unmarried taxpayer over 65. Excluding the benefits leaves only $7,656 of AGI, which produces zero taxable income."
+us,scenario_108,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $10,392 survivor benefits as taxable income and built taxable income from an erroneous $18,048 AGI. Those benefits are excluded from federal AGI, leaving $7,656 and no taxable income after deductions."
+us,scenario_108,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model correctly excluded taxable Social Security but incorrectly added the separate $10,392 survivor benefits to AGI. With federal AGI equal to $7,656 rather than $18,048, the deduction eliminates taxable income and the $35 liability disappears."
 us,scenario_108,federal_refundable_credits,grok-build-0.1,llm_error,age_disability,False,"The model applied the 7.65% childless-EITC phase-in rate without enforcing the childless claimant age limit. At age 85, the head is ineligible for the childless EITC, so the computed $586 EITC must be $0."
 us,scenario_108,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits. The required result is $0 because every refundable-credit component evaluates to zero, including EITC for this 85-year-old claimant with no qualifying children."
 us,scenario_108,free_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -6208,36 +6514,38 @@ us,scenario_108,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identi
 us,scenario_108,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so it failed the required structured-output contract."
 us,scenario_108,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_108,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model denied SNAP solely because net income exceeded 100% of the poverty guideline. It failed to apply categorical eligibility through TANF non-cash assistance, which preserves eligibility and triggers the minimum allotment despite the negative formula benefit."
-us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model applied an ordinary gross-income cutoff and declared the household ineligible. TANF non-cash assistance establishes categorical eligibility, after which the negative calculated allotment is replaced by the one-person minimum."
-us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"The model correctly recognized that the negative formula result leads to a minimum benefit and even computed roughly $276, but then submitted $2,227 without a supporting computation. The applicable monthly minimums aggregate to $287.68, not an independently estimated positive allotment."
-us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model used the ordinary net-income test to deny eligibility, omitting categorical eligibility through TANF non-cash assistance. Its submitted $2,376 also contradicts its own ineligibility conclusion and does not follow from the allotment formula."
-us,scenario_108,snap,claude-opus-5,llm_error,period_annualization,False,"The model identified the minimum-allotment result but annualized a rounded $23 monthly amount as $276. PolicyEngine applies monthly minimums of $23.84 and, after the parameter update, $24.37, which sum to $287.68."
-us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model expressly rejected Wisconsin categorical eligibility and treated failure of the ordinary net-income test as disqualifying. TANF non-cash assistance confers categorical eligibility, so the household receives the minimum allotment rather than zero."
-us,scenario_108,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model asserted net income near $1,700 and a $164 monthly benefit even though its listed deductions leave net income above $2,000 and no excess-shelter deduction. The resulting formula allotment is negative and must be replaced by the minimum allotment, totaling $287.68."
-us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model converted a negative formula allotment directly to zero. For this categorically eligible one-person household, the negative result is replaced by the statutory minimum allotment."
-us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated high gross income and an expected contribution above the maximum allotment as producing no benefit. TANF non-cash categorical eligibility bypasses that denial, and the negative formula amount is raised to the one-person minimum."
-us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The answer implies application of the ordinary income threshold as a complete eligibility test. The household is categorically eligible through TANF non-cash assistance and therefore receives the minimum allotment.
-us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model denied SNAP because gross and net income exceeded ordinary limits. It omitted TANF non-cash categorical eligibility and the minimum-allotment floor that produces $287.68 annually.
-us,scenario_108,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model correctly reached categorical eligibility and the minimum-allotment rule but annualized a rounded $23 per month. The actual monthly minimum changes from $23.84 to $24.37 during the year, producing $287.68."
-us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model treated income above the ordinary limit and a negative formula benefit as requiring zero SNAP. Categorical eligibility through TANF non-cash assistance instead requires application of the one-person minimum allotment.
-us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model treated the net-income limit as dispositive. TANF non-cash assistance makes the household categorically eligible, so the negative calculated benefit is replaced by the minimum allotment."
-us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,The model calculated that 30% of net income exceeded the maximum allotment and stopped at zero. It failed to apply the minimum allotment for this categorically eligible one-person household.
-us,scenario_108,snap,gpt-5.4-mini,llm_error,other,False,"The model supplied $1,172 without deriving net income, the expected contribution, or a monthly allotment. The traced computation produces a negative ordinary allotment and therefore only the monthly minimums, totaling $287.68."
-us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the listed deductions and income as insufficient to establish a positive benefit. TANF non-cash assistance establishes categorical eligibility, and the negative formula result is subject to the minimum-allotment floor."
-us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model correctly found that income could not produce a positive formula allotment but incorrectly set the result to zero. A categorically eligible one-person household receives the minimum allotment when the ordinary formula is negative.
-us,scenario_108,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model estimated a $103 monthly allotment from rent and medical deductions, but rent does not exceed half of adjusted income and therefore creates no excess-shelter deduction. Net income remains $2,035.67, making the formula benefit negative and leaving only the minimum allotment."
-us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model treated failure of the ordinary elderly-or-disabled net-income limit as disqualifying. TANF non-cash categorical eligibility preserves eligibility and leads to the minimum allotment.
-us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model denied the household for exceeding the ordinary net-income limit. It omitted categorical eligibility through TANF non-cash assistance and the resulting minimum-allotment floor.
-us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model treated income limits as the sole eligibility pathway. TANF non-cash assistance makes the household categorically eligible, so it receives the minimum allotment despite its income."
-us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model compared annual net income with 100% of the poverty guideline and denied SNAP. That comparison does not defeat TANF non-cash categorical eligibility, under which the negative formula benefit is replaced by the minimum allotment."
-us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model correctly calculated that ordinary net income exceeds the elderly-or-disabled limit but incorrectly treated that result as disqualifying. TANF non-cash categorical eligibility applies, producing the minimum allotment."
-us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,The model stopped after comparing net income with 100% of the poverty guideline. It omitted categorical eligibility through TANF non-cash assistance and consequently failed to apply the minimum allotment.
-us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated excess net income as eliminating SNAP eligibility. TANF non-cash assistance confers categorical eligibility, so the household receives the one-person minimum allotment."
-us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly found no excess-shelter deduction and an expected contribution above the maximum allotment, but it converted the negative result to zero. Categorical eligibility requires the minimum allotment for this one-person household."
-us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model applied both gross- and net-income limits as disqualifying tests and also omitted the 20% earned-income deduction from its stated net calculation. TANF non-cash categorical eligibility is the controlling pathway, and the resulting negative formula allotment is replaced by the minimum."
-us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model repeatedly recalculated deductions, introduced an unlisted utility allowance, and ultimately converted an expected contribution above the maximum allotment to zero. The household is categorically eligible through TANF non-cash assistance, and the negative formula result is replaced by the minimum allotment."
-us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,The answer implies that the ordinary one-person income limit was treated as dispositive. TANF non-cash categorical eligibility overrides that denial and yields the minimum allotment totaling $287.68.
+us,scenario_108,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model applied the ordinary 100% FPL elderly/disabled net-income test as a disqualifying condition and omitted Wisconsin categorical eligibility through TANF non-cash assistance. It then failed to apply the minimum allotment after the formula produced a negative benefit.
+us,scenario_108,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model disqualified the household under a 130% FPL gross-income test, ignoring the applicable categorical-eligibility pathway and the 200% FPL gross standard reflected in the calculation. Once categorically eligible, the household receives the minimum allotment despite its high net income."
+us,scenario_108,snap,claude-opus-4.7,llm_error,other,False,"The model correctly recognized that the negative formula result triggers a minimum allotment of roughly $23 per month, but then abandoned that computation and submitted $2,227 without a supporting calculation. The applicable monthly minimums sum to $287.68, not an estimated positive allotment near $186 per month."
+us,scenario_108,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model concluded that the household failed the ordinary net-income test and therefore missed categorical eligibility through TANF non-cash assistance. It then submitted $2,376 despite reasoning that the benefit was zero, rather than applying and annualizing the minimum allotment."
+us,scenario_108,snap,claude-opus-5,llm_error,period_annualization,False,"The model identified the minimum-allotment rule but annualized a rounded $23 monthly amount as $276. PolicyEngine applies monthly minimums of $23.84 and $24.37 across the benefit months, which sum to $287.68."
+us,scenario_108,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model expressly denied Wisconsin broad-based categorical eligibility and treated the 100% FPL net-income test as dispositive. The TANF non-cash categorical pathway establishes eligibility, after which the negative formula benefit is replaced by the minimum allotment."
+us,scenario_108,snap,claude-sonnet-5,llm_error,other,False,"The model asserted that deductions reduced net income enough to support a $164 monthly benefit, even though its listed income and deductions leave net income above $2,000 per month and produce a negative maximum-allotment formula. The correct next step is the minimum-allotment floor, not an invented $164 allotment."
+us,scenario_108,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model treated the negative maximum-allotment formula as a zero benefit. Because categorical eligibility applies, the negative result is replaced by the one-person minimum allotment and annualized to $287.68."
+us,scenario_108,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated gross income and a negative formula allotment as grounds for zero SNAP. Gross income is within Wisconsin's applicable categorical limit, and categorical eligibility requires applying the minimum allotment rather than zero."
+us,scenario_108,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,The model used an ordinary income threshold to deny SNAP and omitted categorical eligibility through TANF non-cash assistance. The eligible one-person household receives the statutory minimum allotment.
+us,scenario_108,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model treated excess gross and net income under ordinary tests as dispositive. It missed the TANF non-cash categorical-eligibility pathway and the minimum benefit payable after the regular formula falls below zero.
+us,scenario_108,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model correctly identified categorical eligibility and the minimum-allotment rule but used a flat rounded $23 for all 12 months. The applicable minimum changes across benefit months from $23.84 to $24.37, producing $287.68 annually."
+us,scenario_108,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model used the ordinary income limits and allowed the net-income formula to reduce the benefit to zero. Categorical eligibility applies, so the formula's negative result is floored at the minimum allotment."
+us,scenario_108,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model treated failure of the ordinary net-income threshold as disqualifying. TANF non-cash categorical eligibility preserves eligibility, and the household consequently receives the minimum allotment."
+us,scenario_108,snap,glm-5.2,llm_error,categorical_eligibility,False,"The model computed a negative allotment and stopped at zero. For this categorically eligible one-person household, that result triggers the minimum allotment, whose monthly values sum to $287.68."
+us,scenario_108,snap,gpt-5.4-mini,llm_error,other,False,"The model supplied $1,172 without deriving net income, the maximum-allotment formula, or a monthly benefit. The trace yields a negative regular allotment followed by the minimum-allotment floor, totaling $287.68 rather than $1,172."
+us,scenario_108,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of additional deductions as a basis for zero and omitted categorical eligibility through TANF non-cash assistance. Listed deductions are sufficient to run the formula, and its negative result triggers the minimum allotment."
+us,scenario_108,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model correctly recognized that income produces a negative regular allotment but incorrectly converted that result to zero. Categorical eligibility entitles this one-person household to the minimum allotment.
+us,scenario_108,snap,gpt-5.6-luna,llm_error,other,False,"The model asserted a $103 monthly allotment without reconciling that amount to the household's $2,035.67 net monthly income. Thirty percent of that income exceeds the $298 maximum allotment, so the operative amount is the minimum floor, not $103."
+us,scenario_108,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,The model treated the elderly/disabled net-income limit as dispositive and omitted categorical eligibility through TANF non-cash assistance. Eligibility remains intact and the negative formula amount is replaced by the minimum allotment.
+us,scenario_108,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model denied eligibility solely because countable income exceeded the ordinary elderly/disabled net-income limit. The categorical-eligibility pathway applies, leading to the minimum allotment rather than zero."
+us,scenario_108,snap,grok-4.3,llm_error,categorical_eligibility,False,The model applied ordinary one-person income limits and missed TANF non-cash categorical eligibility. The resulting negative benefit formula is subject to the minimum-allotment floor.
+us,scenario_108,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model treated net income above 100% FPL as an absolute bar. Categorical eligibility overrides that ordinary disqualification here, and the household receives the minimum allotment."
+us,scenario_108,snap,grok-4.6,llm_error,categorical_eligibility,False,"Although the model recognized waiver of the gross test for an elderly/disabled household, it still made the ordinary net-income test dispositive. It omitted TANF non-cash categorical eligibility and the minimum-allotment rule."
+us,scenario_108,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model accurately approximated ordinary net income but incorrectly used the 100% FPL net test to deny the benefit. Categorical eligibility applies, and the negative regular allotment is floored at the monthly minimum."
+us,scenario_108,snap,inkling,llm_error,categorical_eligibility,False,The model stopped after finding net income above the ordinary 100% FPL limit. It omitted categorical eligibility through TANF non-cash assistance and therefore failed to award the minimum allotment.
+us,scenario_108,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated the ordinary elderly/disabled net-income limit as the final eligibility rule. TANF non-cash categorical eligibility establishes eligibility, and the minimum allotment applies."
+us,scenario_108,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model correctly found that 30% of net income exceeds the maximum allotment but incorrectly set the benefit to zero. For the categorically eligible household, the negative formula result is replaced by the one-person minimum allotment."
+us,scenario_108,snap,minimax-m3,llm_error,categorical_eligibility,False,The model applied both an ordinary gross-income test and the 100% FPL net-income test while omitting categorical eligibility through TANF non-cash assistance. That pathway preserves eligibility and produces the minimum allotment.
+us,scenario_108,snap,ox-alpha,llm_error,thresholds_rates,False,"The model applied a 165% FPL gross-income ceiling, while the traced Wisconsin categorical standard accepts this household at 1.98 times the federal poverty guideline. It consequently missed both categorical eligibility and the minimum allotment."
+us,scenario_108,snap,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model repeatedly recomputed deductions, introduced an unlisted utility allowance, and ultimately treated a negative regular allotment as zero despite first saying the household qualifies. With categorical eligibility, the correct final step is to apply the minimum-allotment floor using only listed inputs."
+us,scenario_108,snap,qwen3.8-max,llm_error,categorical_eligibility,False,The model denied SNAP under a generic one-person income limit and omitted the TANF non-cash categorical pathway. The eligible household receives the minimum allotment even though the ordinary benefit formula is negative.
 us,scenario_108,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $10,392 survivor-benefit amount as Wisconsin-taxable income and started from $18,048 of AGI. Excluding that amount leaves only $7,656 of taxable wages, which Wisconsin deductions and exemptions fully offset."
 us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model's $18,048 AGI includes the $10,392 survivor-benefit input as taxable Wisconsin income. That benefit is not part of the Wisconsin taxable base, so the remaining $7,656 of wages is eliminated by Wisconsin deductions and exemptions."
 us,scenario_108,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"Although the model applied a personal exemption and renter-related school property tax credit, it first overstated Wisconsin AGI by including the $10,392 survivor-benefit input. With that nontaxable amount excluded, deductions and exemptions eliminate the tax before the renter credit is needed."
@@ -6245,36 +6553,38 @@ us,scenario_108,state_income_tax_before_refundable_credits,glm-5.2,llm_error,tax
 us,scenario_108,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model explicitly treated the survivor-benefit income as taxable alongside wages. Excluding that $10,392 amount leaves $7,656 of Wisconsin taxable-source income before deductions, and the applicable deductions and exemptions reduce the liability to zero."
 us,scenario_108,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model explicitly included the $10,392 survivor-benefit input in its $18,048 Wisconsin AGI and then applied the standard-deduction phaseout to that overstated base. Excluding the survivor benefits leaves only $7,656 of wages, which is fully sheltered by Wisconsin deductions and exemptions."
 us,scenario_108,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so it failed the required output contract."
-us,scenario_108,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor income by adding the separate $10,392 survivor-benefit input to wages and $12,934 of Social Security, inflating homestead income from $20,590 to $30,982. It then abandoned its own zero-credit conclusion and submitted an unsupported $636 instead of applying the homestead formula to $1,440 of property tax and $20,590 of income."
-us,scenario_108,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model incorrectly excluded Wisconsin's homestead credit from state_refundable_credits because it viewed the program only as property-tax relief. The modeled credit is refundable and the claimant's $1,440 rent-derived property tax and $20,590 homestead income produce $271.39."
-us,scenario_108,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model excluded the $12,934 tax-exempt Social Security amount from homestead income and substituted the separate $10,392 survivor-benefit input, producing the wrong $18,048 income base. The correct base is $20,590, and the exact homestead formula yields $271.39 rather than an estimated $300."
-us,scenario_108,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor income by adding $10,392 to wages and the $12,934 Social Security benefit. Wisconsin homestead income is $20,590, below the applicable limit, and produces a $271.39 credit."
-us,scenario_108,state_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an unexplained income estimate of about $18,000 instead of the traced $20,590 homestead income and did not execute the statutory calculation. Applying the formula to $20,590 of income and $1,440 of rent-derived property tax yields $271.39, not $371."
-us,scenario_108,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed $10,392 survivor-benefit input, inflating homestead income to $30,982. The applicable income is $7,656 plus $12,934, or $20,590, which generates a $271.39 homestead credit."
-us,scenario_108,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model failed to apply the Wisconsin homestead-credit pathway for a claimant who is age 85, disabled, and pays rent. With $20,590 of homestead income and $1,440 of rent-derived property tax, that pathway produces $271.39."
-us,scenario_108,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model reached $30,982 by counting the separate $10,392 survivor-benefit input in addition to wages and Social Security. Homestead income is $20,590, so the claimant remains eligible and receives $271.39."
-us,scenario_108,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly calculated Wisconsin homestead income as $30,982 by double-counting survivor income. The correct $20,590 income base is below the limit and yields a $271.39 refundable homestead credit."
-us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model overlooked the Wisconsin homestead-credit eligibility pathway triggered by the claimant's age, disability, rent, and qualifying income. The traced inputs produce a $271.39 refundable credit."
-us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model inflated homestead income to $30,982 by adding the separate $10,392 survivor-benefit input to wages and Social Security. The correct income is $20,590, which produces a $271.39 credit."
-us,scenario_108,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor income and therefore applied the homestead-credit ceiling to $30,982 instead of $20,590. At $20,590, the claimant qualifies for $271.39."
-us,scenario_108,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the separate $10,392 survivor-benefit amount on top of the $12,934 Social Security survivor benefit. Removing that duplication gives $20,590 of homestead income and a $271.39 credit."
-us,scenario_108,state_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's income-limit conclusion uses an overstated income base. Wisconsin homestead income is $20,590, and the eligible elderly, disabled renter receives $271.39."
-us,scenario_108,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model double-counted the $10,392 survivor-benefit input, causing it to report $30,982 of homestead income and deny the credit. The correct $20,590 base yields $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize that the elderly, disabled Wisconsin renter qualifies for the refundable homestead credit. The $20,590 income and $1,440 rent-derived property-tax inputs generate $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model focused on EITC-type credits and omitted Wisconsin's refundable homestead credit. The claimant's age, disability, rent, and $20,590 homestead income produce $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model treated the separately listed survivor-benefit amount as additional homestead income, pushing the household over the limit. The applicable income is $20,590, and the resulting homestead credit is $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of qualifying children as eliminating every refundable Wisconsin credit. Wisconsin's homestead credit has an elderly or disabled renter pathway, which produces $271.39 here."
-us,scenario_108,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model applied the homestead-credit income limit to an overstated income total. The correct homestead income is $20,590, not $30,982, and it yields $271.39."
-us,scenario_108,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,The model overlooked the refundable Wisconsin homestead credit available to this age-85 disabled renter. The qualifying inputs produce a credit of $271.39.
-us,scenario_108,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted the Wisconsin homestead-credit pathway without applying the claimant's age, disability, rent, and homestead income. Those facts generate a refundable credit of $271.39."
-us,scenario_108,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor income to obtain about $30,982 and incorrectly denied the homestead credit. The correct $20,590 income base produces $271.39; the lack of a Wisconsin EITC does not eliminate that credit."
-us,scenario_108,state_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model added the separate $10,392 survivor-benefit input to wages and Social Security, overstating homestead income as $30,982. The correct $20,590 amount is within the credit range and yields $271.39."
-us,scenario_108,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model stopped after rejecting the Wisconsin EITC and omitted the refundable homestead credit. The elderly, disabled renter qualifies for $271.39 based on $20,590 of homestead income and $1,440 of rent-derived property tax."
-us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no state_refundable_credits value or explanation, violating the required output contract."
-us,scenario_108,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model double-counted the separate $10,392 survivor-benefit input and therefore used about $30,982 as homestead income. The correct $20,590 base produces a $271.39 refundable homestead credit."
-us,scenario_108,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly assumed that zero state income-tax liability prevents a refundable credit. Wisconsin's homestead credit is refundable independently of positive income-tax liability and equals $271.39 here.
-us,scenario_108,state_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double-counted survivor income by adding both the $12,934 Social Security survivor benefit and the separate $10,392 survivor-benefit input. Wisconsin homestead income is $20,590, so the elderly, disabled renter qualifies for $271.39."
-us,scenario_108,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of qualifying children as eliminating all Wisconsin refundable credits. The claimant qualifies for the homestead credit through age, disability, rent, and $20,590 of homestead income, producing $271.39."
+us,scenario_108,state_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the separate $10,392 survivor-benefit input to Wisconsin homestead income, producing $30,982 instead of $20,590. It then contradicted its own zero-credit calculation by submitting an unsupported $636 estimate rather than applying the homestead formula to $1,440 of property tax and $20,590 of income."
+us,scenario_108,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model incorrectly excluded Wisconsin's refundable homestead credit merely because it is property-tax relief. PolicyEngine includes that refundable credit in state_refundable_credits, and this elderly, disabled renter receives $271.39."
+us,scenario_108,state_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $12,934 tax-exempt Social Security benefit from homestead income while including the separate $10,392 survivor-benefit input. The correct income base is $7,656 plus $12,934, or $20,590, which produces $271.39 from $1,440 of property tax."
+us,scenario_108,state_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model double counted benefit income by adding both $12,934 of Social Security survivor benefits and the separate $10,392 survivor-benefit input. Wisconsin homestead income is $20,590, not $30,982, so the income limit does not eliminate the credit."
+us,scenario_108,state_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an income estimate near $18,000 rather than the traced homestead income of $20,590, omitting or substituting benefit income. Applying the formula to $20,590 of income and $1,440 of property tax yields $271.39, not $371."
+us,scenario_108,state_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the $10,392 survivor-benefit input to wages and Social Security when constructing homestead income. The applicable base is $20,590, which remains within the credit calculation and yields $271.39."
+us,scenario_108,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,The model failed to apply the Wisconsin homestead-credit pathway for an 85-year-old disabled renter. The absence of dependents only eliminates Wisconsin EITC; it does not eliminate the $271.39 refundable homestead credit.
+us,scenario_108,state_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model constructed $30,982 of homestead income by incorrectly adding the separate $10,392 survivor-benefit input. The traced income is $20,590, so the homestead credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated $30,982 as Wisconsin homestead income by including the separate survivor-benefit amount. The credit uses $20,590 of income and $1,440 of property tax, producing $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model omitted the Wisconsin homestead credit despite the claimant being 85, disabled, and paying $7,200 in rent. Those facts activate the homestead-credit calculation, which returns $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the separate $10,392 survivor-benefit input in homestead income and therefore compared $30,982 with the limit. The correct $20,590 income base yields a positive $271.39 credit."
+us,scenario_108,state_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model overstated Wisconsin homestead income as $30,982 by adding a benefit input that is not part of the traced base. Homestead income is $20,590, and the resulting refundable credit is $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly used $30,982 as homestead income by summing both benefit fields. The calculation uses wages plus tax-exempt Social Security, totaling $20,590, and returns $271.39."
+us,scenario_108,state_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model's conclusion that income exceeds the homestead limit rests on including the separate $10,392 survivor-benefit input. The applicable income is $20,590, which produces a $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly calculated homestead income as $30,982 by adding both survivor-related entries. The traced base is $20,590, so lack of qualifying children eliminates only EITC and leaves a $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model failed to recognize the refundable Wisconsin homestead credit triggered by the claimant's age, disability, rent, and qualifying income. The homestead calculation returns $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model looked only for an EITC-style refundable credit and omitted Wisconsin's refundable homestead credit. The elderly disabled renter qualifies for $271.39 based on $1,440 of imputed property tax and $20,590 of income."
+us,scenario_108,state_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the separate $10,392 survivor-benefit input when deciding that homestead income exceeded the limit. The correct base is $20,590, and absence of children does not prevent the $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly treated the absence of qualifying children as eliminating all Wisconsin refundable credits. Children are relevant to Wisconsin EITC, while this 85-year-old disabled renter separately qualifies for a $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's above-limit conclusion used an overstated homestead-income base that included the separate $10,392 survivor-benefit input. The applicable income is $20,590, yielding $271.39."
+us,scenario_108,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model omitted the homestead-credit eligibility pathway for an elderly disabled Wisconsin renter. With $20,590 of household income and $1,440 of rent-derived property tax, the credit is $271.39."
+us,scenario_108,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,The model failed to apply Wisconsin's refundable homestead credit to the elderly disabled renter. The listed rent and qualifying income generate $271.39.
+us,scenario_108,state_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly reached roughly $30,982 by adding the separate $10,392 survivor-benefit input to homestead income. Wisconsin EITC is zero, but the correct $20,590 homestead-income base still produces $271.39."
+us,scenario_108,state_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model double counted survivor-related benefits and overstated homestead income as $30,982. Federal EITC being zero affects Wisconsin EITC only; the homestead calculation at $20,590 produces $271.39."
+us,scenario_108,state_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the separate $10,392 survivor-benefit input in Wisconsin homestead income. The relevant total is $20,590, not $30,982, and it yields a $271.39 refundable credit."
+us,scenario_108,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,"The model stopped after determining that Wisconsin EITC was unavailable and omitted the refundable homestead credit. The claimant's age, disability, rent, and $20,590 homestead income produce $271.39."
+us,scenario_108,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for state_refundable_credits.
+us,scenario_108,state_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly added the separate $10,392 survivor-benefit input when calculating broad homestead income. The correct base is $20,590, so zero federal and Wisconsin EITC does not erase the $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model incorrectly assumed that zero state income-tax liability precludes refundable credits. Wisconsin's homestead credit is refundable independently of pre-credit liability and equals $271.39 here.
+us,scenario_108,state_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly included the separate $10,392 survivor-benefit input in homestead income and treated medical adjustments as the only possible reduction. The traced income is $20,590, which produces $271.39 even though Wisconsin EITC is zero."
+us,scenario_108,state_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model double counted survivor-related benefits by adding both $12,934 of Social Security and the separate $10,392 survivor-benefit input to wages. The homestead-credit base is $20,590, and the lack of children affects EITC rather than the $271.39 homestead credit."
+us,scenario_108,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly inferred that no qualifying children meant no applicable Wisconsin refundable credit. The elderly disabled renter qualifies through the separate homestead-credit pathway and receives $271.39.
 us,scenario_109,child1_chip_eligible,claude-haiku-4.5,llm_error,health_coverage,False,"The model checked only CHIP's age and upper-income limits and treated satisfying them as sufficient. It omitted the prior Medicaid screen: the age-9 child qualifies for Florida Medicaid in the OLDER_CHILD category, which makes the child ineligible for CHIP."
 us,scenario_109,child1_chip_eligible,claude-opus-4.7,llm_error,health_coverage,False,"The model calculated MAGI and compared it only with CHIP's upper-income limit. It failed to test the lower Medicaid eligibility pathway first; at this income, the age-9 child is Medicaid-eligible in Florida and therefore excluded from CHIP."
 us,scenario_109,child1_chip_eligible,claude-opus-4.8,llm_error,health_coverage,False,"The model recognized that CHIP applies between Medicaid and CHIP income limits but incorrectly placed this household in that interval. The age-9 child remains within Florida's Medicaid limit for the OLDER_CHILD category, so the child never enters the CHIP coverage band."
@@ -6310,35 +6620,37 @@ us,scenario_109,child3_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_elig
 us,scenario_109,child3_medicaid_eligible,qwen-3.7-max,llm_error,asset_resource,False,"The model correctly found income below an infant Medicaid threshold but then wrongly imposed an asset test and other categorical conditions on the MAGI infant pathway. Florida's lack of adult Medicaid expansion, CHIP availability, and confirmed enrollment are irrelevant because this age-0 dependent qualifies for Medicaid under the infant category at 72% of FPL."
 us,scenario_109,child3_wic_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of an explicitly listed WIC status as disqualifying instead of computing eligibility from the supplied facts. The age-zero child satisfies WIC's child category, and the five-person household's income is below the 185%-of-poverty income limit, so the correct eligibility output is Yes."
 us,scenario_109,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model used an overstated projected standard deduction, incorrectly concluded taxable income and pre-credit tax were zero, and then subtracted an invented $284 of “unused” CTC. The correct computation produces income tax that uses $2,792.91 of the $6,600 CTC as a nonrefundable offset, while the remaining $3,807.09 is refundable; unused nonrefundable credits never make this output negative."
-us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model derived $8,231 of EITC and about $3,807 of refundable CTC, which sum to $12,038, but then rounded the requested dollar amount to $12,000. It discarded $38.09 after completing the correct component calculation."
-us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly declared the family ineligible for EITC even though its own stated three-child married-filing-jointly phaseout endpoint exceeds the household's income. It also replaced its calculated refundable CTC of $3,877.50 with an unsupported $2,500, omitting the $8,231 EITC and miscomputing the child-credit component."
-us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly computed $8,231 of EITC plus $3,807 of refundable CTC, totaling $12,038, and then submitted $10,379 without a corresponding calculation. Its final value contradicts its completed component arithmetic."
-us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model first calculated refundable CTC as $3,781 from net self-employment earnings, then replaced it with an unsupported $4,416. It also used an understated $8,046 EITC instead of $8,231 and failed to use the refundable-CTC earned-income base that produces $3,807.09."
-us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model treated $30,000 as producing a heavily reduced EITC and assigned only $5,035 instead of the full $8,231 three-child credit. It also used gross receipts to calculate $4,125 of refundable CTC rather than the adjusted self-employment earned-income base yielding $3,807.09."
-us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model repeatedly imposed an EITC phaseout at this income instead of awarding the $8,231 three-child EITC. It also calculated refundable CTC from the full $30,000 as $4,125 rather than using the self-employment-derived base that yields $3,807.09, and its submitted $7,594 does not equal any total in its reasoning."
-us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model reduced EITC to $3,015 even though the household receives the full $8,231 three-child credit. It simultaneously asserted that the refundable CTC earned-income limit was satisfied for $5,100 after calculating that limit as only about $3,807, so both components underlying $8,115 were wrong."
-us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,other,False,"The model understated EITC at $8,026 instead of $8,231 and imposed a flat $3,000 refundable CTC instead of applying the earned-income formula, which yields $3,807.09. Those two component errors produced $11,026."
-us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an inaccurate $8,290 EITC estimate and an obsolete $1,000-per-child refundable CTC cap. The applicable components are $8,231 of EITC and $3,807.09 from the refundable-CTC earned-income formula."
-us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model assigned only $3,588 of EITC and $1,674 of refundable CTC without applying the governing three-child EITC maximum or refundable-CTC earned-income formula. Those rules produce $8,231 and $3,807.09, respectively."
-us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model estimated the EITC as $8,250 rather than using $8,231 and capped the Additional Child Tax Credit at a flat $3,000. The refundable-CTC earned-income computation produces $3,807.09, making total refundable credits $12,038.09."
-us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an estimated $8,240 EITC and an obsolete $1,000-per-child ACTC amount. The correct components are the $8,231 EITC and $3,807.09 refundable CTC determined by the earned-income limitation."
-us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model understated EITC by using $8,219 and fixed refundable CTC at $3,000. Applying the 2026 EITC amount and refundable-CTC earned-income formula gives $8,231 and $3,807.09."
-us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The unexplained $11,046 total omits $992.09 from the two applicable components. The household receives $8,231 of EITC and $3,807.09 of refundable CTC, totaling $12,038.09."
-us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an understated estimated EITC maximum of $8,025.75 instead of $8,231. It also based refundable CTC on $27,705 rather than the applicable earned-income amount after the deductible half of self-employment tax, understating that component as $3,780.75 instead of $3,807.09."
-us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model supplied a coarse unsupported $13,400 estimate without calculating either refundable credit. The actual component computation is $8,231 of EITC plus $3,807.09 of refundable CTC, so its estimate overstates the total by $1,361.91."
-us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model ignored that the listed $30,000 of self-employment income is earned income for both EITC and refundable CTC purposes. With three qualifying children, that income produces an $8,231 EITC and $3,807.09 refundable CTC rather than zero."
-us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly used the $8,231 EITC but limited the Additional Child Tax Credit to an obsolete $1,000 per child. The applicable earned-income limitation yields $3,807.09 of refundable CTC, not $3,000."
-us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model correctly used $8,231 of EITC but calculated refundable CTC as 15% of gross $30,000 above $2,500. The refundable-CTC earned-income base reflects the deductible half of self-employment tax and yields $3,807.09 rather than $4,125."
-us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $8,231 EITC but used net earnings subject to self-employment tax, $27,705, as the refundable-CTC earned-income base. The applicable base after the deductible half of self-employment tax produces $3,807.09, not about $3,781."
-us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model correctly used $8,231 of EITC but computed $4,125 of refundable CTC from gross self-employment receipts. Accounting for the deductible half of self-employment tax in the relevant earned-income base yields $3,807.09."
-us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted both refundable credits despite $30,000 of self-employment earned income and three qualifying children. Those facts produce an $8,231 EITC and $3,807.09 refundable CTC."
-us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model imposed a post-TCJA $1,000-per-child ACTC cap and used a $3,000 earned-income floor, producing only $3,000 of refundable CTC. The applicable refundable-CTC calculation uses the $2,500 threshold and yields $3,807.09, while EITC is $8,231 rather than $8,226."
-us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model capped refundable CTC at an obsolete $1,000 per child and used a $3,000 income threshold, limiting the credit to $3,000 instead of $3,807.09. It also estimated EITC at $8,245 rather than applying the $8,231 amount."
-us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model calculated refundable CTC at approximately the correct $3,807 amount but used an inflated estimated EITC maximum of $8,271. The applicable EITC is $8,231, and retaining the precise refundable CTC produces $12,038.09."
-us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The $6,695 answer omits $5,343.09 from the two refundable-credit components. The correct derivation awards $8,231 of EITC and $3,807.09 of refundable CTC; no component combination in that derivation produces $6,695."
-us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly phased the three-child EITC down to $3,588 instead of awarding $8,231 and treated refundable CTC as the full $5,100 per-child-cap total instead of the lower $3,807.09 earned-income limit. Its submitted $4,288 also contradicts the $8,688 sum stated in its own reasoning."
-us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model repeatedly applied an EITC phaseout at thresholds below the applicable married-joint threshold and ultimately substituted an unsupported $5,475 EITC for the $8,231 credit. It also used gross $30,000 income to calculate $4,125 of refundable CTC instead of the adjusted self-employment earned-income base yielding $3,807.09."
-us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the Child Tax Credit as entirely nonrefundable and therefore omitted the Additional Child Tax Credit. This household receives $3,807.09 of refundable CTC in addition to the $8,231 EITC; the model also understated EITC as $8,046."
+us,scenario_109,federal_refundable_credits,claude-fable-5,llm_error,other,False,"The model correctly derived $8,231 of EITC and about $3,807 of refundable CTC, then improperly rounded their $12,038 total to $12,000. The requested output required the calculated annual amount, not rounding to the nearest thousand."
+us,scenario_109,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model falsely excluded EITC by treating income below the roughly $50,000-plus phaseout endpoint as exceeding the eligible range. It also replaced its own $3,877.50 ACTC calculation with an unsupported $2,500 amount instead of combining $8,231 of EITC with $3,807.09 of refundable CTC."
+us,scenario_109,federal_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model explicitly computed the correct components—$8,231 of EITC and $3,807 of ACTC—and their $12,038 sum, but then submitted an unrelated $10,379 estimate. Its final value discarded its completed arithmetic."
+us,scenario_109,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used $27,705 of net earnings from self-employment for ACTC and then replaced the resulting $3,781 with an unexplained $4,416. ACTC uses the applicable earned-income amount after the deductible half of self-employment tax here, producing $3,807.09, while the 2026 EITC is $8,231."
+us,scenario_109,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly reduced the three-child EITC to about $5,035 at $30,000 of self-employment income. The household receives the full $8,231 EITC, and ACTC is $3,807.09 after the self-employment-tax deduction rather than $4,125."
+us,scenario_109,federal_refundable_credits,claude-sonnet-4.6,llm_error,credit_phaseout,False,"The model repeatedly applied obsolete or invented EITC maxima and phaseout thresholds, although the household receives the full 2026 maximum of $8,231. It also used gross $30,000 income for ACTC instead of the earned-income amount after deducting half of self-employment tax, overstating ACTC as $4,125 rather than $3,807.09, and its submitted $7,594 did not equal any final component total it derived."
+us,scenario_109,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model reduced EITC to $3,015 using a phaseout that does not apply and misunderstood the ACTC cap as permitting $5,100 despite its own lower earned-income limit. The correct components are $8,231 of EITC and $3,807.09 of ACTC."
+us,scenario_109,federal_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model understated both components by using $8,026 instead of the 2026 $8,231 EITC and a flat $3,000 ACTC. The ACTC earned-income formula produces $3,807.09."
+us,scenario_109,federal_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used an incorrect approximate EITC maximum of $8,290 and an obsolete flat $1,000-per-child ACTC. The applicable amounts are $8,231 and $3,807.09."
+us,scenario_109,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model reduced EITC to $3,588 even though this three-child joint household receives the full $8,231 credit. It also understated the earned-income-limited ACTC as $1,674 instead of $3,807.09."
+us,scenario_109,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model approximated the EITC maximum as $8,250 and imposed a flat $3,000 ACTC. The 2026 amounts are $8,231 of EITC and $3,807.09 under the ACTC earned-income formula."
+us,scenario_109,federal_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an approximate $8,240 EITC and an obsolete $1,000-per-child refundable CTC cap. The calculation requires $8,231 of EITC plus $3,807.09 of earned-income-limited ACTC."
+us,scenario_109,federal_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model used $8,219 rather than the $8,231 2026 EITC and limited refundable CTC to $3,000 without applying the ACTC earned-income formula. That formula yields $3,807.09."
+us,scenario_109,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The unexplained $11,046 total omits $992.09 from the two applicable components. The engine calculation is $8,231 of EITC plus $3,807.09 of refundable CTC."
+us,scenario_109,federal_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an outdated estimated EITC maximum of $8,025.75 instead of $8,231. It also based ACTC on $27,705 of net self-employment earnings rather than the applicable earned-income amount after the deductible half of self-employment tax, producing $3,780.75 instead of $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model supplied a coarse unsupported estimate instead of applying the two credit formulas. The actual components are $8,231 of EITC and $3,807.09 of ACTC, so $13,400 overstates their total by $1,361.91."
+us,scenario_109,federal_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of wages as the absence of earned income and ignored the explicitly listed $30,000 of self-employment income. Self-employment income supports both the $8,231 EITC and $3,807.09 ACTC."
+us,scenario_109,federal_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly used the $8,231 EITC but imposed an obsolete flat $1,000-per-child ACTC. The ACTC earned-income limitation yields $3,807.09, not $3,000."
+us,scenario_109,federal_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model correctly used $8,231 of EITC but calculated ACTC from gross $30,000 income, yielding $4,125. Deducting half of self-employment tax in the applicable earned-income computation reduces ACTC to $3,807.09."
+us,scenario_109,federal_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used net earnings from self-employment of about $27,705 as the ACTC base and obtained about $3,781. The applicable earned-income computation after the deductible half of self-employment tax yields $3,807.09, which combines with the correctly identified $8,231 EITC."
+us,scenario_109,federal_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model correctly used $8,231 of EITC but applied the ACTC formula to gross $30,000 income. The applicable income after the deductible half of self-employment tax produces $3,807.09 rather than $4,125."
+us,scenario_109,federal_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model omitted both refundable credits despite $30,000 of self-employment earned income and three qualifying children. Those facts generate an $8,231 EITC and $3,807.09 refundable CTC."
+us,scenario_109,federal_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied obsolete post-TCJA parameters—a $1,000-per-child ACTC and a $3,000 income floor—and used an incorrect $8,226 EITC maximum. The 2026 calculation produces $8,231 of EITC and $3,807.09 of ACTC."
+us,scenario_109,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child CTC, a $3,000 ACTC threshold, and an incorrect $8,247 EITC maximum. The applicable amounts are $8,231 of EITC and $3,807.09 of ACTC."
+us,scenario_109,federal_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an incorrect $8,245 EITC maximum and obsolete ACTC rules limiting the credit to $1,000 per child above a $3,000 floor. The correct components are $8,231 and $3,807.09."
+us,scenario_109,federal_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model calculated ACTC at approximately the correct $3,807 amount but used an inflated EITC maximum of about $8,271. The 2026 three-child maximum is $8,231, making the total $12,038.09."
+us,scenario_109,federal_refundable_credits,kimi-k2.6,llm_error,other,False,"The $6,695 answer omits $5,343.09 from the two refundable components. The household receives $8,231 of EITC and $3,807.09 of refundable CTC; the submitted amount does not implement that combination."
+us,scenario_109,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly phased the EITC down to $3,588 instead of awarding the full $8,231 and incorrectly treated the ACTC per-child cap as automatically payable. ACTC is instead constrained by the earned-income formula to $3,807.09, and its submitted $4,288 is also inconsistent with its stated $8,688 component sum."
+us,scenario_109,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model invented an approximately $24,000 EITC phaseout start and reduced a full $8,231 EITC to about $7,414. It also treated the $5,100 aggregate ACTC cap as fully refundable even though the earned-income limit is only $3,807.09; its stated reasoning and submitted amount are internally inconsistent as well."
+us,scenario_109,federal_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model applied several conflicting estimated EITC phaseout thresholds and ultimately substituted a conservative $5,475 figure instead of the full $8,231 credit. It also used gross $30,000 income for ACTC, producing $4,125 rather than $3,807.09 after the self-employment-tax deduction."
+us,scenario_109,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model treated the Child Tax Credit as entirely nonrefundable and therefore omitted the Additional Child Tax Credit. This household receives $3,807.09 of refundable CTC in addition to the $8,231 EITC."
 us,scenario_109,free_school_meals_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of explicitly stated school-meal receipt as ineligibility instead of calculating eligibility from the household facts. It missed both automatic categorical qualification through $8,020.55 of annual SNAP benefits and direct income qualification at 78% of the federal poverty guideline, below the 130% free-meal limit."
 us,scenario_109,head_medicaid_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly treated Florida as offering Medicaid expansion to low-income adults and therefore used low MAGI as sufficient for eligibility. Florida has no expansion pathway for this age-30 head, and the head fits no other categorical pathway, producing medicaid_category NONE and ineligibility."
 us,scenario_109,head_wic_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly transferred the infant's WIC categorical eligibility to the Head by treating parent or caretaker status in a WIC-eligible family as a qualifying category. WIC has no general parent/caretaker pathway; the Head lacks every listed adult categorical status and is therefore ineligible.
@@ -6350,36 +6662,38 @@ us,scenario_109,self_employment_tax,gpt-5.4-mini,llm_error,thresholds_rates,Fals
 us,scenario_109,self_employment_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model correctly settled on applying 15.3% to 92.35% of income but evaluated $30,000 × 0.9235 × 0.153 incorrectly. That product is $4,238.865, not $4,234; no Additional Medicare Tax adjustment changes this result."
 us,scenario_109,self_employment_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model stated the correct base and rate but performed an unsupported rounding to $4,241. The calculation produces $4,238.865 and therefore $4,238.86 in the benchmark output."
 us,scenario_109,self_employment_tax,minimax-m3,llm_error,thresholds_rates,False,"The model correctly calculated the $27,705 net self-employment earnings base, but rounded the Social Security and Medicare components prematurely and then submitted $4,233 even though its own rounded components sum to $4,238. Computing the combined rate without premature rounding gives $4,238.865."
-us,scenario_109,snap,claude-fable-5,llm_error,other,False,"The model's own first calculation produced roughly $7,500 annually, but it replaced that result with $3,768 based on an unexplained higher-income adjustment. SNAP does not add self-employment or payroll-tax adjustments that raise the stated $2,500 monthly income, and the applicable monthly schedules yield $8,020.55."
-us,scenario_109,snap,claude-haiku-4.5,llm_error,other,False,"The model calculated SNAP from federal tax AGI and an unexplained $244 monthly benefit instead of using the SNAP earned-income and standard deductions. The correct net income is $1,739 for January–September and $1,733.15 for October–December, producing $8,020.55 annually."
-us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $1,613 maximum monthly allotment for a five-person household, far above the applicable $1,183 and $1,209.52 amounts. Applying the correct fiscal-year maximum allotments and deductions yields $8,020.55."
-us,scenario_109,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model improperly reduced SNAP income using a 7.65% self-employment-tax adjustment and then treated rent as generating a shelter deduction. With the 20% earned-income deduction and standard deduction, rent remains below half of adjusted income, so no excess-shelter deduction applies."
-us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,"The asserted $424 monthly benefit does not follow from the stated income and allowable deductions. Net monthly income is $1,739 for nine months and $1,733.15 for three months, against maximum allotments of $1,183 and $1,209.52."
-us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used maximum allotments of $1,017–$1,034 instead of $1,183 for January–September and $1,209.52 for October–December. It also held one schedule constant for all twelve months rather than applying the October fiscal-year update."
-us,scenario_109,snap,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model counted six household members even though the household contains two adults and three children, and it simultaneously claimed $2,309 was above a stated $4,590 gross-income limit. The five-person household passes the income tests and receives $8,020.55."
-us,scenario_109,snap,deepseek-v4-pro,llm_error,period_annualization,False,"The model substituted a single $232 deduction and $1,184 allotment for all twelve months. The calculation requires the $261/$1,183 schedule for nine months and the $266.85/$1,209.52 schedule for three months."
-us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used rounded net income and an unsupported $695 monthly benefit instead of the applicable deductions, maximum allotments, and expected contributions. The two fiscal-year schedules produce nine payments near $661.30 and three near $689.62."
-us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies a flat $765 monthly allotment without identifying the maximum allotment or 30% expected contribution. The traced schedules instead produce $8,020.55 for the year."
-us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction even though $800 rent is below half of adjusted monthly income under both fiscal-year schedules. With no shelter deduction, monthly benefits are about $661.30 for nine months and $689.62 for three months."
-us,scenario_109,snap,gemini-3.5-flash,llm_error,period_annualization,False,"The model rounded net income to $1,750 and applied one $665 monthly result throughout the year. It omitted the October updates to both the standard deduction and maximum allotment and failed to sum the unrounded monthly amounts."
-us,scenario_109,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $800 monthly rent as a deduction from SNAP income. Rent produces no excess-shelter deduction because it does not exceed half of adjusted income, so the annual benefit is $8,020.55."
-us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,The answer implies $732 per month but supplies no computation supporting that amount. The applicable net-income contribution and maximum-allotment schedules yield about $661.30 for nine months and $689.62 for three months.
-us,scenario_109,snap,glm-5.2,llm_error,period_annualization,False,"The model used one approximate $236 standard deduction and $1,155 maximum allotment for every month. SNAP instead uses the January–September schedule and a higher October–December schedule, summed without prematurely rounding monthly benefits."
-us,scenario_109,snap,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer invokes high shelter costs even though $800 rent is below half of adjusted income and creates no excess-shelter deduction. Applying only the earned-income and standard deductions produces $8,020.55."
-us,scenario_109,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model imposed a no-take-up assumption despite the prompt explicitly requiring program take-up when needed. The supplied income, assets, and household facts are sufficient to establish eligibility and calculate $8,020.55."
-us,scenario_109,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly derived the January–September monthly amount but multiplied a rounded $661 by twelve. It omitted the October increase in the standard deduction and maximum allotment, which raises the final three monthly benefits to about $689.62."
-us,scenario_109,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The answer applies an unspecified single monthly allotment across the year. The October fiscal-year parameter update must be incorporated, and the unrounded monthly benefits sum to $8,020.55."
-us,scenario_109,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model multiplied the January–September rounded benefit of $661 by all twelve months. The last three months use the $266.85 standard deduction and $1,209.52 maximum allotment, producing about $689.62 monthly."
-us,scenario_109,snap,gpt-5.6-terra,llm_error,period_annualization,False,The model applied a flat $659 monthly amount across all twelve months. It omitted the separate January–September and October–December parameter schedules and their unrounded monthly calculations.
-us,scenario_109,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared the household over SNAP income and asset limits. The five-person household with $2,500 monthly earned income and $900 in assets passes the income, asset, and work-requirement tests throughout the year."
-us,scenario_109,snap,grok-4.5,llm_error,period_annualization,False,"The model projected one $1,213 maximum allotment and one $690 benefit for every month. The applicable maximum is $1,183 for nine months and $1,209.52 for three months, with corresponding changes to the standard deduction."
-us,scenario_109,snap,grok-build-0.1,llm_error,period_annualization,False,"The model used a single $254 deduction and $1,189 maximum allotment for all twelve months. The correct month-specific parameters are $261 and $1,183 through September, then $266.85 and $1,209.52 from October."
-us,scenario_109,snap,inkling,llm_error,period_annualization,False,"The model rounded the standard deduction, net income, and maximum allotment into one $665 monthly figure. It omitted the October fiscal-year update and the required summation of unrounded monthly benefits."
-us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used a $300 monthly standard deduction and a $1,220 maximum allotment rather than the applicable fiscal-year parameters. The standard deductions are $261 and $266.85, while the maximum allotments are $1,183 and $1,209.52."
-us,scenario_109,snap,kimi-k3,llm_error,period_annualization,False,"The model accurately reproduced the January–September calculation but extended its rounded $661 result through December. October–December use updated parameters and pay about $689.62 monthly, bringing the annual total to $8,020.55."
-us,scenario_109,snap,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model asserted net income near zero even though the listed deductions leave $1,739 monthly through September and $1,733.15 thereafter. The $800 rent creates no excess-shelter deduction because it is below half of adjusted income."
-us,scenario_109,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model effectively awarded a near-maximum allotment after treating the 20% earned-income deduction as eliminating the benefit reduction. Countable net income remains $1,739 or $1,733.15 monthly, and 30% of that income must be subtracted from the maximum allotment."
-us,scenario_109,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used a four-person maximum for a household containing five people and incorrectly reduced countable net income to zero. The earned-income and standard deductions leave positive net income, whose 30% expected contribution must reduce the five-person maximum allotment."
+us,scenario_109,snap,claude-fable-5,llm_error,other,False,"The model correctly reached roughly $7,500 from its stated deduction and allotment assumptions, then submitted $3,768 using an unexplained higher-income adjustment that contradicts its own calculation. SNAP does not replace the trace’s $2,500 monthly self-employment income with that unsupported adjustment."
+us,scenario_109,snap,claude-haiku-4.5,llm_error,thresholds_rates,False,"The model used an unsupported $244 monthly benefit despite stating net income near $1,830 and a maximum allotment of $1,316, which do not produce $244 under the maximum-allotment-minus-30%-of-net-income formula. It also substituted tax AGI and a half-SE-tax deduction into a SNAP calculation where the trace applies the 20% earned-income deduction and SNAP standard deduction to $2,500 monthly income."
+us,scenario_109,snap,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $1,613 maximum monthly allotment for five people instead of the applicable $1,183 from January through September and $1,209.52 from October through December. That inflated parameter directly produced its $13,016 estimate."
+us,scenario_109,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model improperly reduced SNAP self-employment income by 7.65% for self-employment tax and then invoked a shelter deduction even though $800 rent is below half of adjusted income. The trace instead leaves $1,739 before October and $1,733.15 afterward, with no excess-shelter deduction."
+us,scenario_109,snap,claude-opus-5,llm_error,thresholds_rates,False,The model’s $424 monthly estimate does not apply the applicable five-person maximum allotments to net income after the 20% earned-income and standard deductions. Those inputs yield about $661.30 for January–September and $689.62 for October–December.
+us,scenario_109,snap,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated maximum allotments of $1,017–$1,034 instead of $1,183 for January–September and $1,209.52 for October–December. It also held an estimated standard deduction constant rather than applying the $261 and $266.85 schedules."
+us,scenario_109,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model miscounted the five listed people as a household of six and asserted that $2,309 was above a stated $4,590 gross-income limit, reversing its own comparison. The five-person household passes the income, asset, and work-requirement tests and receives positive SNAP."
+us,scenario_109,snap,deepseek-v4-pro,llm_error,period_annualization,False,"The model used approximate constant values of a $232 deduction and $1,184 allotment for all twelve months. It missed the January–September $261/$1,183 schedule and the October–December $266.85/$1,209.52 update."
+us,scenario_109,snap,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model approximated net income as $1,750 and the benefit as $695 without applying the traced deduction and maximum-allotment parameters. The exact schedules yield about $661.30 for nine months and $689.62 for three months."
+us,scenario_109,snap,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $765 monthly average lacks the required maximum-allotment-minus-30%-of-net-income computation. With traced net income and parameters, the weighted monthly average is about $668.38, totaling $8,020.55."
+us,scenario_109,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied an excess-shelter deduction even though $800 monthly rent is below half of adjusted income under both parameter schedules. Removing that nonexistent deduction leaves net income of $1,739 and $1,733.15, not the lower income implicit in a $786 benefit."
+us,scenario_109,snap,gemini-3.5-flash,llm_error,period_annualization,False,The model used rounded estimated net income and a constant $665 benefit for all twelve months. It missed the October parameter update that raises the monthly benefit to about $689.62 after nine months at about $661.30.
+us,scenario_109,snap,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the $800 monthly rent as producing a deduction that raises the benefit to $752.50 per month. Rent is below half of adjusted income, so the excess-shelter deduction is zero and the traced benefits are about $661.30 and $689.62."
+us,scenario_109,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model’s $732 monthly average does not follow from the applicable deductions, contribution rate, and five-person maximum allotments. Those parameters produce a weighted monthly average of about $668.38."
+us,scenario_109,snap,glm-5.2,llm_error,period_annualization,False,"The model used approximate constant values of a $236 standard deduction and $1,155 maximum allotment. It failed to apply the $261/$1,183 schedule for nine months and the $266.85/$1,209.52 schedule for the final three months."
+us,scenario_109,snap,gpt-5.4-mini,llm_error,thresholds_rates,False,The model’s unexplained estimate of about $484.67 per month understates the result of the SNAP benefit formula. The traced deductions and allotments produce about $661.30 for January–September and $689.62 for October–December.
+us,scenario_109,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model imposed a no-take-up assumption even though the prompt explicitly requires program take-up when needed and supplies all inputs for calculation. The household passes every SNAP eligibility test and therefore cannot be assigned zero.
+us,scenario_109,snap,gpt-5.5,llm_error,period_annualization,False,"The model correctly computed the January–September monthly amount but multiplied that schedule by twelve. It omitted the October–December increase in the standard deduction and maximum allotment, which raises those three monthly benefits to about $689.62."
+us,scenario_109,snap,gpt-5.6-luna,llm_error,period_annualization,False,"The model collapsed the year into an estimated constant $659 monthly allotment. SNAP parameters change in October, so the calculation requires nine months at about $661.30 and three months at about $689.62."
+us,scenario_109,snap,gpt-5.6-sol,llm_error,period_annualization,False,"The model applied the January–September $661 rounded benefit to all twelve months. It omitted the October–December fiscal-year update, when the traced benefit rises to about $689.62 per month."
+us,scenario_109,snap,gpt-5.6-terra,llm_error,period_annualization,False,The model used one estimated $659 monthly value for the full calendar year. It failed to split 2026 between the nine-month old schedule and three-month updated schedule.
+us,scenario_109,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model incorrectly declared the household over SNAP income and asset limits. Monthly income is $2,500, bank assets are only $900, and the household passes the income, asset, and work-requirement tests throughout the year."
+us,scenario_109,snap,grok-4.5,llm_error,period_annualization,False,"The model projected a single $1,213 maximum allotment and $690 benefit across all twelve months. The applicable maximum is $1,183 for nine months and $1,209.52 for three months, paired with their respective standard deductions."
+us,scenario_109,snap,grok-4.6,llm_error,period_annualization,False,"The model replaced the two in-year schedules with one approximate $1,190 maximum allotment and annualized a rounded $668 monthly benefit. Exact month-by-month computation and summation yields $8,020.55."
+us,scenario_109,snap,grok-build-0.1,llm_error,period_annualization,False,"The model used one approximate deduction and allotment for the entire year, despite labeling the deduction inconsistently as $254 while deriving $1,740. It omitted the October schedule change and the unrounded monthly summation."
+us,scenario_109,snap,inkling,llm_error,period_annualization,False,"The model used approximate constant values of $250 for the standard deduction, $1,190 for the maximum allotment, and $665 for the benefit. The actual parameters change in October and must be summed as nine months at about $661.30 plus three at about $689.62."
+us,scenario_109,snap,kimi-k2.6,llm_error,thresholds_rates,False,"The model used an invented $300 monthly standard deduction and a projected $1,220 maximum allotment. The trace uses standard deductions of $261 and $266.85 and maximum allotments of $1,183 and $1,209.52 in their respective months."
+us,scenario_109,snap,kimi-k3,llm_error,period_annualization,False,The model correctly reproduced the January–September calculation but annualized its rounded $661 result. It omitted the October–December parameter update and failed to sum unrounded monthly benefits.
+us,scenario_109,snap,minimax-m3,llm_error,other,False,"The model asserted net income was approximately zero, then acknowledged a $1,151 maximum allotment and submitted $489 per month without a coherent benefit calculation. The applicable deductions leave positive net income of $1,739 before October and $1,733.15 afterward, producing the traced contributions and benefits."
+us,scenario_109,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model invented a Florida utility allowance even though unlisted expenses must be zero, creating an excess-shelter deduction that does not apply. Its submitted $7,920 also contradicts the $9,400 conclusion in its explanation; with rent alone, excess shelter costs are zero and the two traced schedules total $8,020.55."
+us,scenario_109,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model treated $2,000 after the earned-income deduction as the gross-income-test amount and then effectively assigned a near-maximum $962 benefit without subtracting 30% of traced net income. The benefit must subtract contributions of $521.70 and $519.90 from the applicable maximum allotments."
+us,scenario_109,snap,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model used a four-person maximum allotment even though the household contains two adults and three children. It also incorrectly reduced countable net income to zero; the traced net income is $1,739 before October and $1,733.15 afterward."
 us,scenario_109,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented a federal minimum Medicaid pathway for any household member with no individual income and few assets. Florida requires this nondisabled adult to satisfy a covered category, and Medicaid MAGI eligibility uses household income rather than treating the spouse’s unlisted personal income as an isolated $0 eligibility base."
 us,scenario_109,spouse_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model correctly calculated that household income exceeds Florida’s parent/caretaker limit and explicitly concluded that the spouse is not eligible, but then submitted value 1. No alternative Medicaid category applies to reverse its own eligibility analysis."
 us,scenario_109,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model improperly converted the presence of an infant into pregnancy or postpartum eligibility. An age-zero child does not establish that the spouse is pregnant, postpartum, or the child’s mother, and caretaker status alone fails because household income exceeds Florida’s parent/caretaker threshold."
@@ -6393,36 +6707,38 @@ us,scenario_109,spouse_wic_eligible,claude-sonnet-5,llm_error,categorical_eligib
 us,scenario_109,spouse_wic_eligible,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model assumed the spouse was the postpartum mother of the infant. Because postpartum status is not listed, it is false under the prompt, leaving the spouse outside every adult WIC category."
 us,scenario_109,spouse_wic_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred postpartum or breastfeeding status from the spouse's age and the infant's presence. Neither fact establishes the spouse's own qualifying status, and the prompt explicitly makes those unlisted statuses false."
 us,scenario_109,spouse_wic_eligible,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model inferred recent postpartum status or current breastfeeding from the existence of a zero-year-old child. Those personal statuses were unlisted and therefore false, so the spouse fails categorical eligibility before the income threshold affects the result."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income directly to AGI and used only a projected $16,100 standard deduction, omitting the traced $17,752.06 of taxable-income deductions, including the $1,264.59 QBI deduction. Its submitted $21,120 also contradicts its own completed calculation of about $25,841."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included the estate-income input in gross income, used a $15,000 standard deduction, and omitted the traced QBI deduction, leaving taxable income far above $145,669.58. It then subtracted an unexplained $1,300 nonrefundable credit despite identifying no applicable credit."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the $6,323 estate-income input as taxable federal income and used only a $16,100 standard deduction, omitting the $1,264.59 QBI deduction and the full traced deduction computation. This inflated ordinary taxable income from the traced amount underlying $17,764.88 of ordinary tax to $112,959."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income to AGI and reduced taxable income only by a projected $16,100 standard deduction. It omitted the traced $17,752.06 deduction total, including the $1,264.59 QBI deduction, and therefore overstated ordinary-rate tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated components support tax near $25,758, not its submitted $30,150. It both omitted the traced deduction treatment and made an internal arithmetic/output error by converting roughly $20,000 of ordinary tax plus $6,133 of preferential tax into $30,150."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included $6,323 of estate income in AGI, selected an estimated $15,750 standard deduction, and omitted the $1,264.59 QBI deduction. These choices produced $154,193 of taxable income instead of the traced $145,669.58."
-us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model improperly built deductions around restored miscellaneous employee-expense treatment and omitted the traced QBI deduction. More decisively, its own calculation produced about $26,045, but it submitted $20,200 after an unsupported adjustment despite correctly finding no NIIT and no credits."
-us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied a pre-TCJA-style $6,800 standard deduction, personal exemption, deductible employee expenses, and 10%–28% ordinary brackets instead of the applicable 2026 computation. It also subtracted employer-sponsored premiums from wages and then overstated ordinary tax to $32,490."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used a personal exemption and a miscellaneous itemized deduction for unreimbursed employee expenses while omitting the traced $1,264.59 QBI deduction. It also included the estate-income input in AGI, so its taxable-income construction did not reach $145,669.58."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model's abbreviated derivation omitted the $6,323 estate-income field entirely but also failed to apply the traced $198.35 above-the-line deduction and $17,752.06 taxable-income deductions, including QBI. Its $19,688 result understates the $17,764.88 ordinary tax plus $6,132.56 preferential tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted employer health premiums from wages and invoked personal exemptions under an assumed TCJA sunset. The traced computation instead uses $95,755.10 of employment income and a $17,752.06 deduction total containing a $1,264.59 QBI deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and constructed deductions from a personal exemption, estimated state tax, and unreimbursed employee expenses. It omitted the traced QBI deduction and consequently calculated the wrong ordinary taxable-income base and $20,928 of ordinary tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model correctly approached the traced AGI only by subtracting employer health premiums from wages, which is not the traced wage calculation. It then failed to apply the specific $17,752.06 deduction total and the resulting split of $17,764.88 ordinary tax plus $6,132.56 capital-gains tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The answer started from an approximate $163,054 AGI rather than the traced $163,421.64 and never identified the applicable deductions. Applying the traced $17,752.06 deduction total yields $145,669.58 of taxable income and $23,897.44 of tax, not $22,765."
-us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income to AGI and used only a $15,400 estimated standard deduction. It omitted the traced $17,752.06 deduction total, including the $1,264.59 QBI deduction, thereby overstating ordinary taxable income and tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The explanation explicitly says federal income tax remains positive, but the model submitted zero. The traced ordinary and preferential computations total $17,764.88 plus $6,132.56, and no credit reduces that sum to zero."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model supplied no reproducible income or deduction calculation and its $16,811 is below the traced ordinary tax alone. It failed to add the $6,132.56 tax on $40,883.77 of net capital gains and qualified dividends after computing the ordinary-income portion."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income to AGI and used only a $16,100 standard deduction. It omitted the traced deduction total of $17,752.06, including the $1,264.59 QBI deduction, which inflated ordinary taxable income to $112,959."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model relied on a $387 nonitemizer charitable deduction but did not apply the traced $17,752.06 taxable-income deduction total or identify the $1,264.59 QBI deduction. Its resulting taxable-income base therefore differs from the traced $145,669.58."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income to AGI and reduced income only by a $16,100 standard deduction. It omitted the traced $1,264.59 QBI deduction and the remainder of the $17,752.06 deduction computation, overstating ordinary-rate tax."
-us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used $169,943 of AGI and only a $16,100 standard deduction. The traced calculation uses $163,421.64 of AGI and $17,752.06 of taxable-income deductions, including $1,264.59 of QBI deduction, so the model taxed too much ordinary income."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model used a generic itemized-deduction estimate and unspecified nonrefundable credits instead of the traced deduction and tax computation. The applicable calculation produces $17,764.88 of ordinary tax and $6,132.56 of preferential tax with no stated credit offset."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model assumed restored pre-TCJA brackets, a personal exemption, and a miscellaneous deduction for unreimbursed employee expenses. It also included estate income in AGI and omitted the traced QBI deduction, so both its taxable-income base and ordinary rates were wrong."
-us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used post-sunset 10%/15%/25% brackets, a personal exemption, and a miscellaneous employee-expense deduction. It also added estate income to AGI and substituted estimated SALT deductions for the traced $17,752.06 deduction total containing the QBI deduction."
-us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model added estate income to AGI and deducted estimated Ohio tax plus unreimbursed employee expenses, while omitting the traced $1,264.59 QBI deduction. Its $21,146 itemized estimate therefore replaced the actual deduction pathway and produced the wrong ordinary taxable income."
-us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric output or explanation for the requested variable. The required structured answer was therefore missing.
-us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented an $8,425.50 overtime deduction by estimating an overtime premium from hours and wages, despite the prompt defining gross wages as the complete annual total and providing no separate deductible overtime amount. It also used a nonitemizer charitable deduction rather than the traced $17,752.06 deduction computation."
-us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted that PolicyEngine returned zero without performing the income-tax calculation. The traced ordinary tax of $17,764.88 and preferential tax of $6,132.56 produce a positive $23,897.44 liability."
-us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model omitted the $4,244.90 pre-tax 401(k) reduction, the $198.35 above-the-line deduction, and the traced $17,752.06 taxable-income deductions. Its prose nevertheless calculates about $26,973, so the submitted $43,879.20 is also an unsupported output inconsistent with its own arithmetic."
-us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model mis-summed the income components into $113,617 of AGI, used a $297.87 QBI deduction instead of $1,264.59, and invented a $194.77 nonrefundable credit despite identifying no qualifying credit. These errors replaced the traced $163,421.64 AGI and $17,752.06 taxable-income deduction computation."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income directly to AGI, denied the $198.35 above-the-line deduction, and omitted the $1,264.59 QBI deduction. Those errors produced $153,843 of taxable income instead of $145,669.58 and overstated ordinary-income tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model included the estate-income input in gross income and omitted the $1,264.59 QBI deduction and the full $17,752.06 taxable-income deduction. It then invented an unexplained $1,300 nonrefundable credit despite identifying no applicable credit."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model treated the $6,323 estate-income input as an AGI inclusion, disallowed the $198.35 adjustment, and omitted the $1,264.59 QBI deduction. Its approximate brackets were consequently applied to ordinary taxable income $7,289 above the traced amount."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"Although it allowed roughly the correct IRA adjustment, the model still added $6,323 of estate income to AGI and omitted the QBI deduction and full taxable-income deductions. This left taxable income $7,975 above $145,669.58."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's own components yield about $25,758, but it submitted $30,150 after describing an internally inconsistent combination of ordinary and preferential tax. It also included estate income in AGI and omitted the QBI deduction, so even its intermediate taxable income was too high."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model added estate income to AGI, rejected the $198.35 adjustment, omitted the $1,264.59 QBI deduction, and used projected rather than applicable 2026 deduction and bracket parameters. These choices overstated both taxable and ordinary taxable income."
+us,scenario_110,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model first computed about $26,045, then replaced it with $20,200 using nonexistent unspecified adjustments. It also included estate income in AGI, omitted QBI, and incorrectly invoked a restored miscellaneous employee-expense deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used obsolete post-sunset concepts—a $6,800 standard deduction, personal exemption, deductible employee expenses, and 28% ordinary rates—instead of the applicable 2026 rules. It also subtracted employer-sponsored insurance premiums from wages without the trace supporting that additional exclusion."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and applied a personal exemption plus a miscellaneous employee-business-expense deduction that do not produce the traced deductions. It omitted the $198.35 adjustment and $1,264.59 QBI deduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model omitted the $6,323 estate input and all reference-specific deduction detail from its explanation, but its $19,688 result is far below the $17,764.88 ordinary tax plus $6,132.56 preferential tax. It failed to correctly separate and tax the $40,883.77 of net capital gains and qualified dividends while deriving taxable income."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted employer health premiums from wages and applied personal exemptions under a TCJA-sunset assumption. The trace instead starts from $95,755.10 of employment income and reaches $145,669.58 through the specified above-the-line and taxable-income deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and substituted estimated state-tax, employee-expense, and personal-exemption deductions for the traced $17,752.06 deduction. It also omitted the $198.35 above-the-line adjustment and QBI treatment."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced wages by employer-sponsored health premiums to obtain its $163,054 AGI. The traced AGI is $163,421.64 after $95,755.10 of employment income and the $198.35 adjustment, followed by $17,752.06 of taxable-income deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model used an unexplained $163,054 AGI rather than the traced $163,421.64 and did not account for the $17,752.06 deduction composition, including QBI. Its result therefore does not reproduce the $17,764.88 ordinary tax plus $6,132.56 preferential tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model added $6,323 of estate income directly to AGI and omitted both the $198.35 adjustment and $1,264.59 QBI deduction. It also used a $15,400 estimated standard deduction instead of the traced total taxable-income deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The explanation explicitly states that federal income tax remains positive but submits zero. The traced computation yields $17,764.88 of ordinary tax and $6,132.56 of capital-gains tax, with no credits reducing them to zero."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The submitted $16,811 is below the traced ordinary-income tax alone and therefore omits or offsets the $6,132.56 preferential tax without any applicable nonrefundable credit. Its generic explanation supplies no computation supporting that reduction."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model added estate income directly to AGI, omitted the $198.35 adjustment and $1,264.59 QBI deduction, and used only a $16,100 standard deduction. This produced $153,843 of taxable income rather than $145,669.58."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model accounted for the cash-charity deduction but still treated the income base and remaining deductions incorrectly. It failed to reach the traced $163,421.64 AGI and to include the $1,264.59 QBI deduction within total deductions of $17,752.06."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model added estate income directly to AGI and used only a $16,100 standard deduction. It omitted the $198.35 above-the-line adjustment, the QBI deduction, and the rest of the traced $17,752.06 taxable-income deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model's $169,943 AGI directly includes estate income and omits the $198.35 adjustment. Its $16,100 deduction also omits the cash-charity and $1,264.59 QBI components that reduce taxable income to $145,669.58."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model invoked unspecified itemized deductions and nonrefundable credits rather than deriving the traced AGI and deductions. No such credit explains its reduction; the liability is the sum of $17,764.88 ordinary tax and $6,132.56 preferential tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model added estate income to AGI and incorrectly restored a personal exemption and miscellaneous employee-expense deduction. It omitted the $198.35 adjustment and $1,264.59 QBI deduction and applied the wrong ordinary bracket regime."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied restored 10%, 15%, and 25% pre-TCJA brackets together with a personal exemption and employee-expense deduction. The applicable computation uses the traced deductions and current main rates, producing $17,764.88 of ordinary tax."
+us,scenario_110,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model included estate income in AGI and substituted an estimated state-income-tax deduction, employee-expense deduction, and personal exemption for the traced deduction structure. Although it allowed an IRA deduction, it omitted the $1,264.59 QBI deduction and used the wrong ordinary brackets."
+us,scenario_110,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,The model included estate income in AGI and used estimated Ohio income tax plus a miscellaneous employee-expense deduction. It omitted the $198.35 adjustment and QBI deduction and reported taxable income inconsistent with its own subtraction from AGI.
+us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable output or reasoning for the requested variable.
+us,scenario_110,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model invented an $8,425.50 overtime deduction by estimating an overtime premium from the hourly-rate facts, even though the annual gross wage is the controlling income input and the trace contains no such deduction. It also used only a $297.87 QBI deduction instead of $1,264.59, producing the wrong taxable and ordinary-income amounts."
+us,scenario_110,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted a zero engine result without performing the supplied income-tax computation. The traced ordinary and preferential components total $23,897.44, and no nonrefundable credit eliminates that liability."
+us,scenario_110,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model included estate income directly in AGI and omitted the $198.35 adjustment and $1,264.59 QBI deduction. Adding only the $387 nonitemizer charity deduction left taxable income $7,786 above the traced $145,669.58."
+us,scenario_110,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model's written arithmetic totals about $26,973, but it submitted $43,879.20 with no supporting computation. Its intermediate work also failed to subtract the traditional 401(k), included estate income directly, and omitted the IRA and QBI deductions."
+us,scenario_110,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model mis-summed the listed income into a $113,617 AGI, used only a $297.87 QBI deduction, and invented a $194.77 nonrefundable credit despite identifying no qualifying credit. The correct sequence gives AGI of $163,421.64, taxable income of $145,669.58, and no such credit reduction."
 us,scenario_110,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_110,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model deducted the $6,889 employer-sponsored insurance premium from FICA wages even though this input does not reduce the $100,000 payroll-tax base in this calculation. Applying employee Social Security and Medicare tax to the full wages yields $7,650."
 us,scenario_110,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA wages by the $6,889 employer-sponsored insurance premium. Payroll tax applies to the full $100,000 wage input here, producing $6,200 of Social Security tax and $1,450 of Medicare tax."
@@ -6437,36 +6753,38 @@ us,scenario_110,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The mod
 us,scenario_110,payroll_tax,qwen-3.7-max,llm_error,other,False,"The model’s explanation correctly calculated $6,200 of Social Security tax plus $1,450 of Medicare tax and explicitly concluded $7,650, but it submitted the unrelated value $9,351.90. The numeric output therefore failed to carry the model’s own final calculation into the required value field."
 us,scenario_110,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model wrongly imposed Additional Medicare Tax by treating nonwage household income as wages exceeding the $200,000 threshold. Only the $100,000 wage amount enters that threshold test, so Additional Medicare Tax is zero and total employee payroll tax is $7,650."
 us,scenario_110,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model computed Ohio AGI as roughly $169,745 instead of $163,421.64 and then produced a final $5,052 that contradicts its own refined $4,265.55 calculation. It therefore combined an incorrect income base with an unsupported final-number substitution."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly subtracted a federal-style $15,000 standard deduction in calculating Ohio taxable income. Ohio's traced calculation subtracts only the $1,900 personal exemption from Ohio AGI of $163,421.64, leaving $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model first derived approximately $4,328 from its assumed income and brackets, then replaced that result with $2,900 based on unspecified “federal items and exemption.” That unsupported second adjustment has no place in the Ohio computation."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model applied a 3.5% top rate to nearly all income above $26,050 and used an Ohio AGI near $169,745 rather than $163,421.64. It also submitted $4,517 despite its own arithmetic producing roughly $5,305 after the stated credit."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model started from an overstated AGI near $169,700 and incorrectly subtracted $954 of tax-exempt interest from that AGI. The traced Ohio base is $163,421.64, followed by only the $1,900 personal exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented an Ohio itemized deduction of $10,915 for mortgage interest, property tax, and donations. The traced calculation does not deduct those items; it subtracts the $1,900 personal exemption from $163,421.64 and then applies the Ohio brackets."
-us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model used obsolete graduated rates reaching 3.5% and 3.99%, added tax-exempt interest without establishing the required Ohio modification, and invoked nonexistent retirement credits despite no retirement income. It failed to apply the 2026 schedule to traced taxable income of $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model taxed its entire assumed income at 2.75%, ignored the $1,900 personal exemption, and asserted that Ohio had no personal exemption. The correct bracket computation uses $161,521.64 of taxable income and includes the schedule's bracket base rather than multiplying all AGI by one rate."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and used a $2,500 exemption instead of the traced $1,900 exemption. It consequently applied the brackets to $167,443 rather than $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer invokes unspecified “standard deductions,” but the traced Ohio computation subtracts only the $1,900 personal exemption from $163,421.64. Its $2,981 therefore rests on an excessive or otherwise incorrect reduction of the Ohio tax base."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used $163,054 as federal AGI instead of the traced Ohio AGI of $163,421.64 and did not account for the $1,900 personal exemption in its explanation. The Ohio schedule must be applied to $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model overstated AGI as $169,943 and applied a rate structure with an approximately 3.5% top rate. The required calculation instead applies the 2026 Ohio schedule to $161,521.64 after the $1,900 exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $4,240.52 does not result from applying the Ohio brackets to the traced taxable income of $161,521.64. It reflects an incorrect tax base, bracket parameters, or both."
-us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted $3,986 does not follow from the traced sequence of $163,421.64 Ohio AGI, a $1,900 exemption, and the 2026 bracket calculation. Its bare reference to Ohio taxable income omits the bracket step that yields $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and incorrectly eliminated the personal exemption through a claimed phaseout. The traced calculation uses Ohio AGI of $163,421.64 and deducts a $1,900 exemption before applying the brackets."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model explicitly recognized that Ohio tax was positive but submitted zero because it declined to estimate the state computation. Applying the supplied household facts through the Ohio AGI, exemption, and bracket steps produces a positive $4,057.47 liability."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated mortgage interest and charitable contributions as relevant Ohio itemized deductions and still produced an excessive $5,625 estimate. The traced Ohio computation instead uses $163,421.64 of AGI, subtracts only the $1,900 personal exemption, and taxes $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model overstated Ohio AGI as $169,943 and calculated only 2.75% of income above $26,050. That shortcut omits the Ohio schedule's accumulated bracket-base tax and uses the wrong taxable-income starting point."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The unexplained $4,418 does not equal the result of applying the 2026 Ohio schedule to $161,521.64. The answer therefore used an incorrect rate schedule or tax base despite referring generally to the exemption and rates."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model calculated 2.75% only on income above the zero-tax threshold and treated that marginal amount as total tax. It omitted the schedule's bracket-base amount and did not use the traced $161,521.64 taxable income."
-us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model used an overstated AGI of $169,943, an incorrect $1,850 exemption, and 2.75% of only the excess over $26,050. It omitted the bracket-base tax required by the Ohio schedule."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The model supplied no computation supporting its $3,200 estimate. Applying the Ohio schedule to the traced $161,521.64 taxable income yields $4,057.47, so the answer embodies an incorrect base, rate schedule, or unsupported credits."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used 2024-style graduated brackets including a 3.5% rate and overstated Ohio AGI as roughly $169,943. The 2026 schedule must instead be applied to $161,521.64 after the traced $1,900 exemption."
-us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used an incorrect $169,745 AGI and $2,400 exemption, then computed only 2.75% of income above $26,050. This both misstates taxable income and omits the schedule's accumulated bracket-base tax."
-us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model overstated Ohio AGI to roughly $169,943 and applied a 3.5% rate above $100,000. The required 2026 bracket calculation instead starts with taxable income of $161,521.64 and produces $4,057.47."
-us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no numeric value or explanation for the requested output. It therefore failed the required output contract before any substantive Ohio tax calculation could be evaluated.
-us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model used an overstated $169,745 AGI, computed 2.75% only on income above $26,050, and then subtracted an unsupported $20 personal-exemption credit. It omitted the Ohio schedule's bracket-base tax and used the wrong taxable income."
-us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted a zero PolicyEngine result without performing the Ohio AGI, exemption, or bracket calculation. The traced steps produce taxable income of $161,521.64 and a positive $4,057.47 liability."
-us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model invented a $250 personal-exemption credit and an additional unspecified “graduated personal income tax credit” to reduce its computed tax to $3,148.93. It also used estimated obsolete brackets instead of applying the 2026 schedule to $161,521.64."
-us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model reduced Ohio taxable income to approximately $96,517 through unspecified standard deductions and exemptions. The traced Ohio taxable income is $161,521.64 because only the $1,900 personal exemption is subtracted from $163,421.64 of Ohio AGI."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model constructed Ohio AGI as approximately $169,745 instead of $163,421.64 and used the wrong exemption amount. Its submitted $5,052 also contradicts its own refined calculation of $4,265.55, adding a separate arithmetic/output error."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly deducted a federal-style $15,000 standard deduction in computing Ohio taxable income. Ohio taxable income here is $163,421.64 minus the $1,900 Ohio personal exemption, or $161,521.64."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model first derived an overstated AGI and obsolete bracket result near $4,328, then replaced it with an unsupported $2,900 estimate. It never applied the traced calculation of $163,421.64 AGI less the $1,900 exemption through the 2026 schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an overstated $169,745 AGI, the wrong personal exemption, and a 3.5% rate formula. The applicable computation taxes $161,521.64 under the 2026 Ohio schedule and yields $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model began from an overstated AGI near $169,700 and incorrectly treated tax-exempt interest as an Ohio subtraction from that amount. Ohio AGI is $163,421.64 and taxable income after the personal exemption is $161,521.64."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented an Ohio choice between a standard deduction and itemized deductions and deducted $10,915 of federal-style itemized expenses. It also applied an obsolete 3.5% formula and a $20 exemption credit instead of taxing the traced $161,521.64 under the 2026 schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model added tax-exempt interest to an already overstated AGI and used obsolete graduated brackets, including rates above 3.5%. It also invoked unspecified retirement and other credits despite no qualifying retirement income."
+us,scenario_110,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model asserted a flat 2.75% tax on all income with no personal exemption. Ohio subtracts the $1,900 exemption and applies its rate schedule to $161,521.64 rather than multiplying an invented $164,008 base by 2.75%."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and used a $2,500 exemption instead of the traced $163,421.64 AGI and $1,900 exemption. It therefore applied the brackets to the wrong taxable-income base."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an unsupported standard-deduction calculation. The required base is Ohio AGI of $163,421.64 less only the applicable $1,900 personal exemption before applying the Ohio schedule."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"Although the model cited an AGI close to the correct starting point, it did not subtract the $1,900 exemption and apply the 2026 Ohio schedule correctly. That sequence produces taxable income of $161,521.64 and tax of $4,057.47, not $4,241."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an overstated $169,943 AGI and unspecified exemptions. The traced Ohio base is $161,521.64 after subtracting the $1,900 exemption from $163,421.64."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The unexplained $4,240.52 result does not implement the traced bracket computation. Applying the 2026 Ohio schedule to $161,521.64 produces $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model supplied no derivation and its number implies the wrong rate or threshold computation. Ohio taxable income is $161,521.64, and the applicable schedule produces $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and incorrectly eliminated the personal exemption through a phaseout. The applicable exemption is $1,900, leaving $161,521.64 taxable, and its two-rate bracket schedule was also not the operative 2026 computation."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model knowingly replaced a positive Ohio liability with zero because it declined to estimate the state computation. The supplied facts yield $163,421.64 of Ohio AGI, $161,521.64 after the exemption, and $4,057.47 of tax."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated mortgage interest and charitable contributions as relevant Ohio itemized deductions and produced an unsupported approximation. The traced calculation subtracts the $1,900 personal exemption from Ohio AGI and does not use those expenses in this tax base."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 instead of $163,421.64. Although it used the correct $1,900 exemption, applying 2.75% above $26,050 to its erroneous base produced the wrong tax."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The terse result does not match the applicable 2026 Ohio rate computation. The schedule applied to $161,521.64 yields $4,057.47, not $4,418."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,thresholds_rates,False,"The model reduced the entire amount above the zero-tax threshold to a single 2.75% multiplication. The applicable 2026 Ohio schedule applied to taxable income of $161,521.64 produces $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and used a $1,850 exemption rather than $1,900. It then applied an oversimplified 2.75% formula to the wrong base."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model gave only an unsupported approximate liability and did not calculate the traced tax base or schedule. Ohio AGI of $163,421.64 less the $1,900 exemption yields $161,521.64 and tax of $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an overstated $169,943 AGI and an obsolete 2024-style two-rate schedule. The 2026 calculation starts at $163,421.64 and taxes $161,521.64 after the exemption."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used the correct $1,900 exemption but subtracted it from an overstated $169,943 AGI. It also incorrectly treated 2.75% of all income above $26,050 as the complete 2026 tax formula."
+us,scenario_110,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as $169,745, used a $2,400 exemption rather than $1,900, and applied a simplified 2.75% formula. The correct taxable base is $161,521.64."
+us,scenario_110,state_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model used an overstated taxable base near $167,500 and an obsolete split-rate approximation with 3.5% above $100,000. The applicable 2026 schedule applied to $161,521.64 yields $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required structured result was missing."
+us,scenario_110,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model overstated AGI as $169,745 and then subtracted an inapplicable $20 personal-exemption credit after already deducting the exemption. The traced computation uses $163,421.64 of AGI, a $1,900 exemption, and no such extra credit in reaching $4,057.47."
+us,scenario_110,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model falsely stated that the Ohio computation returned zero and supplied no tax calculation. The Ohio-only liability is $4,057.47 after applying the rate schedule to $161,521.64."
+us,scenario_110,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model overstated Ohio AGI as $169,943 and used a $1,700 exemption instead of $1,900. It then applied a simplified flat 2.75% formula to the resulting wrong base."
+us,scenario_110,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,credit_phaseout,False,"The model included tax-exempt interest in its federal AGI, used estimated obsolete brackets, and invented a $250 personal exemption credit plus an unspecified graduated credit. The traced calculation instead subtracts a $1,900 exemption from $163,421.64 and applies the 2026 schedule without those fabricated credits."
+us,scenario_110,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model reduced Ohio taxable income to $96,517 using unspecified standard deductions and exemptions. The actual Ohio taxable income is $161,521.64 because the traced calculation subtracts only the $1,900 personal exemption from $163,421.64."
 us,scenario_110,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_111,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated about $16,600 of Social Security as taxable even though its own stated combined income of $30,940 is below the $32,000 joint-filer threshold, not above the $44,000 upper threshold. It then calculated zero taxable income after the standard deduction but replaced the resulting $0 tax with an unsupported $1,362 estimate."
 us,scenario_111,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly included $5,405.50 of Social Security in AGI, but still correctly found that the standard deduction reduced taxable income and tax liability to $0. It then submitted $1,400 despite deriving no taxable income and identifying no tax or credit computation that produces that amount."
@@ -6475,13 +6793,14 @@ us,scenario_111,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eli
 us,scenario_111,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model assumed that blind and disabled status automatically opened an SSI-related Medicaid pathway despite zero SSI receipt, then treated listed medical expenses as a spend-down that established eligibility. PolicyEngine assigns no aged/blind/disabled or other Medicaid category to the head, so the resource-limit and vehicle-exclusion discussion never establishes eligibility."
 us,scenario_111,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that the household has no wages, then contradicted that reasoning by imposing $1,046.85 of mandatory payroll tax on retirement income. Social Security retirement benefits and taxable IRA distributions are not employee payroll-tax bases, so the absence of employment compensation yields payroll_tax of $0."
 us,scenario_111,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated income below Washington's 200% gross-income screen as categorical eligibility and then awarded the $24 monthly minimum. The household fails the applicable income eligibility determination, so it is ineligible rather than an eligible household whose calculated allotment is raised to the minimum."
-us,scenario_111,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model treated blindness or disability as conferring Washington Medicaid eligibility regardless of income. Those statuses only open disability-related pathways; the spouse still fails their applicable eligibility tests and has medicaid_category NONE.
-us,scenario_111,spouse_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly isolated the spouse's zero personal income and disregarded the income attributed under the applicable Medicaid household and spousal-income rules. The resulting household income is $43,640, and neither the 138% FPL expansion pathway nor a disability-based pathway applies."
-us,scenario_111,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model asserted that blindness and disability satisfy the disability-related pathway without applying its financial eligibility test. At $43,640 of household income and 2.02 times FPL for MAGI purposes, the spouse qualifies through neither ABD nor adult-expansion Medicaid."
-us,scenario_111,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model converted potential access to SSI-related or medically needy pathways into automatic eligibility based on blindness and disability. The spouse receives no SSI and fails every available Medicaid pathway, so disability does not override the failed financial tests."
-us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,asset_resource,False,"The model concluded that resources were below the ABD limit by excluding the vehicle, then incorrectly declared the household's $43,640 income low enough for ABD Medicaid. Passing an asserted resource test does not cure the failed disability-pathway income test, and the spouse qualifies through no category."
-us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model assumed unspecified exclusions reduce Social Security and IRA income enough to pass the ABD income test and then treated the spouse's share of income as independently qualifying. The applicable calculations leave the spouse outside both the disability-related pathways and MAGI expansion, with MAGI income at 2.02 times FPL."
-us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model reduced household MAGI to the $18,240 IRA distribution by omitting the income included in the engine's Medicaid calculation, producing a false result below 138% FPL. The engine calculation places the spouse at 2.02 times FPL, and zero personal income plus resources below an asserted limit does not independently establish ABD eligibility."
+us,scenario_111,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated blindness or disability as conferring Medicaid eligibility regardless of income. Those statuses only open disability-related pathways, and the spouse does not satisfy any pathway’s remaining eligibility requirements."
+us,scenario_111,spouse_medicaid_eligible,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly assigned the spouse zero countable income under spousal-income rules and used that assertion to approve an ABD/SSI-related pathway. The applicable eligibility calculations leave the spouse in no Medicaid category, while household MAGI is 2.02 times FPL and fails the expansion pathway."
+us,scenario_111,spouse_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model jumped from the spouse’s blindness and disability to qualification under an ABD pathway without applying that pathway’s income and other eligibility tests. Disability creates a potential category, not automatic coverage, and the spouse qualifies through none of the available categories."
+us,scenario_111,spouse_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model acknowledged that household income exceeds the ACA expansion limit but then treated blindness or disability as sufficient for SSI-related or medically needy eligibility. Neither status waives the separate requirements of those pathways, and the spouse receives no SSI and qualifies through no Medicaid category."
+us,scenario_111,spouse_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model asserted that the head’s Social Security and IRA income was low enough for the blind/disabled income limit without calculating or applying that limit. Excluding one vehicle and satisfying a resource ceiling does not cure failure of the applicable income and categorical tests, so no ABD pathway is established."
+us,scenario_111,spouse_medicaid_eligible,kimi-k2.6,llm_error,thresholds_rates,False,"The model characterized countable income as low after unspecified exclusions and then approved ABD Medicaid without computing the controlling income test. The spouse’s disability and low stated resources do not independently establish eligibility, and no ABD or other non-MAGI category is satisfied."
+us,scenario_111,spouse_medicaid_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model understated Medicaid MAGI by counting only the $18,240 IRA distribution and excluding all Social Security on the ground that none was federally taxable. The eligibility computation produces MAGI equal to 2.02 times FPL, which is above the 138% FPL adult expansion limit."
+us,scenario_111,spouse_medicaid_eligible,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model treated the spouse’s lack of personally assigned income as zero income for ABD Medicaid and also understated household MAGI as $18,240. The applicable calculations yield MAGI of 2.02 times FPL and do not place the spouse in an ABD or any other Medicaid category; excluding the vehicle and noting $840 of bank assets does not establish eligibility."
 us,scenario_111,spouse_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model treated disability as automatically conferring Medicare eligibility regardless of age. The spouse is 49, and the prompt supplies no Social Security disability entitlement or other qualifying under-65 Medicare pathway, so the spouse is not Medicare eligible."
 us,scenario_111,spouse_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model collapsed the under-65 Medicare pathway into a bare disability test. Disability alone is insufficient, and the spouse has no listed Social Security disability entitlement or other qualifying Medicare basis."
 us,scenario_111,spouse_medicare_eligible,claude-opus-4.8,llm_error,age_disability,False,"The model acknowledged that disability can lead to Medicare through Social Security disability benefits but then inferred that entitlement from the disability flag. Because unlisted benefit receipt and status are false and no Social Security disability entitlement is provided, the 49-year-old spouse does not qualify."
@@ -6525,66 +6844,70 @@ us,scenario_112,self_employment_tax,claude-haiku-4.5,llm_error,payroll_tax_base,
 us,scenario_112,self_employment_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model correctly stated that passive real-estate farm rent is not subject to self-employment tax, then contradicted that rule by calculating tax as though the entire $1,920 were farm-operator self-employment earnings. The actual net self-employment tax base is zero because the household has no positive net self-employment earnings."
 us,scenario_112,self_employment_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model improperly combined farm rent, financial assistance, and the capital loss into a self-employment tax base. Financial assistance and capital gains or losses are not self-employment earnings, and the household’s relevant net earnings are nonpositive, so no self-employment tax applies."
 us,scenario_112,self_employment_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model treated the full $1,920 of farm rent as net self-employment income and applied the standard 92.35% and 15.3% factors without establishing positive net earnings from self-employment. The resulting tax base is zero, and its submitted $269 also does not match its own stated calculation of about $271."
-us,scenario_112,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the ordinary 130% gross-income screen as disqualifying and failed to apply categorical eligibility through TANF non-cash assistance. It also understated the applicable SNAP deductions, producing net income of about $1,429 instead of $1,269.19, which passes the net-income test."
-us,scenario_112,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model disqualified the household under the ordinary gross-income limit without applying its categorical eligibility through TANF non-cash assistance. Its asset discussion also incorrectly compared a vehicle with the liquid-resource ceiling instead of following the categorical-eligibility pathway.
-us,scenario_112,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,The model stopped at the ordinary 130% gross-income test and never applied categorical eligibility through TANF non-cash assistance. The household consequently proceeds to and passes the net-income test.
-us,scenario_112,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $13,000 financial-assistance input from SNAP income and therefore calculated an artificially low expected contribution and a large allotment. Counting that assistance and applying the engine's SNAP deductions yields $1,269.19 monthly net income and only the minimum allotment."
-us,scenario_112,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,000 financial assistance from countable SNAP income, reducing monthly income to roughly $654 and producing a near-maximum benefit. With that assistance included, the expected contribution exceeds the maximum allotment and the eligible household receives only the minimum allotment."
-us,scenario_112,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model applied the ordinary 130% gross-income test as an absolute bar and failed to apply categorical eligibility through TANF non-cash assistance. The vehicle and ordinary asset-limit analysis does not disqualify a categorically eligible household.
-us,scenario_112,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated gross income above 130% of the poverty guideline as dispositive and omitted the TANF non-cash categorical-eligibility pathway. It also asserted that no deductions offset income even though the engine's deductions reduce monthly net income to $1,269.19, below the net limit."
-us,scenario_112,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model ended the analysis at the ordinary 130% gross-income screen. TANF non-cash assistance establishes categorical eligibility, after which the household passes the applicable net-income test."
-us,scenario_112,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model recognized expanded categorical eligibility but computed SNAP net income as $1,440.20 by omitting applicable deductions. The engine's monthly net income is $1,269.19, which is below the $1,304.17 limit."
-us,scenario_112,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The zero answer implies that the model did not apply categorical eligibility through TANF non-cash assistance and the passing $1,269.19 net-income test. Those steps establish eligibility and trigger the annual minimum allotment."
-us,scenario_112,snap,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model treated an expected contribution above the maximum allotment as producing no benefit. SNAP instead pays the minimum allotment to this eligible one-person household, totaling $287.68 for the year."
-us,scenario_112,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model both overstated net income at about $1,440 and floored the allotment at zero when 30% of net income exceeded the maximum. The correct net income is $1,269.19, and an eligible one-person household receives the statutory minimum allotment rather than zero."
-us,scenario_112,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly converted an expected contribution above the maximum allotment into a zero benefit. Because the household is eligible, the SNAP minimum-allotment rule produces $287.68 annually."
-us,scenario_112,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that net income exceeds the eligibility limit. Applicable SNAP deductions reduce monthly net income to $1,269.19, below the $1,304.17 limit, after which the minimum allotment applies."
-us,scenario_112,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used only the standard and earned-income deductions and therefore overstated monthly net income at about $1,434. The full SNAP deduction calculation yields $1,269.19, which passes the $1,304.17 net-income test."
-us,scenario_112,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The zero answer ignores categorical eligibility through TANF non-cash assistance and the passing net-income test. Once eligible, the household receives the minimum allotment even though its expected contribution exceeds the maximum allotment."
-us,scenario_112,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The prompt supplied a one-person household and all facts needed for the SNAP calculation, so defaulting to zero for allegedly missing indicators was erroneous. TANF non-cash categorical eligibility and $1,269.19 monthly net income establish eligibility and the minimum allotment."
-us,scenario_112,snap,gpt-5.5,llm_error,categorical_eligibility,False,The model treated the ordinary gross-income ceiling as disqualifying and omitted categorical eligibility through TANF non-cash assistance. The household instead proceeds to and passes the net-income test.
-us,scenario_112,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly made the ordinary one-person gross-income limit dispositive. TANF non-cash assistance confers categorical eligibility, and the household's net income passes the remaining income test."
-us,scenario_112,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's no-shelter-deduction calculation omitted deductions used in the SNAP net-income computation and therefore declared a failed net test. The complete calculation produces $1,269.19 monthly net income, below the $1,304.17 limit."
-us,scenario_112,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model applied the ordinary gross-income limit without the TANF non-cash categorical-eligibility pathway. Gross income therefore does not disqualify this household, which passes the net-income test."
-us,scenario_112,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model's generic income-and-assets rejection omitted categorical eligibility through TANF non-cash assistance. The household also has only $10 in bank assets and passes the net-income test, so the minimum allotment is payable."
-us,scenario_112,snap,grok-4.5,llm_error,categorical_eligibility,False,"The model relied on either the ordinary gross limit or an overstated net-income figure. TANF non-cash assistance bypasses the ordinary gross screen, and the correct $1,269.19 net income passes the $1,304.17 limit."
-us,scenario_112,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model disqualified the household solely because $20,843 exceeded its estimated ordinary gross-income limit. It failed to apply categorical eligibility through TANF non-cash assistance, under which the household proceeds to a passing net-income test."
-us,scenario_112,snap,inkling,llm_error,categorical_eligibility,False,The model treated the 130% gross-income limit as the only eligibility route. TANF non-cash categorical eligibility prevents that screen from disqualifying the household.
-us,scenario_112,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model applied the ordinary gross-income requirement for a nonelderly, nondisabled household and omitted categorical eligibility through TANF non-cash assistance. The household therefore remains eligible after passing the net-income test."
-us,scenario_112,snap,kimi-k3,llm_error,categorical_eligibility,False,The model explicitly denied TANF categorical eligibility even though the household qualifies through TANF non-cash assistance. It also treated an expected contribution above the maximum allotment as eliminating benefits instead of applying the minimum allotment.
-us,scenario_112,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred ineligibility merely from a one-person household having wages and performed none of the controlling eligibility calculations. TANF non-cash assistance establishes categorical eligibility, and $1,269.19 of monthly net income passes the net-income test."
-us,scenario_112,snap,qwen-3.7-max,llm_error,thresholds_rates,False,"The model's submitted $3,576 contradicts its own calculation that countable assistance makes the formula allotment nonpositive, and it then substituted an unsupported near-maximum estimate. The nonpositive formula result is subject to the minimum-allotment rule, producing $287.68 annually rather than a near-maximum benefit."
-us,scenario_112,snap,qwen3.8-max,llm_error,categorical_eligibility,False,The unsupported zero answer omits categorical eligibility through TANF non-cash assistance and the passing net-income test. Those rules make the household eligible for the annual minimum allotment.
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions by using only listed real-estate tax for SALT and calculating medical and charitable deductions differently, producing $23,624 before QBI instead of the combined $28,892.88 itemized-plus-senior deductions. It then abandoned its own $11,889 bracket calculation and submitted $8,971 without a valid tax computation."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from AGI, chose an invented $28,300 standard deduction instead of the beneficial itemized deductions, and failed to subtract QBI consistently. It also added employee Medicare tax and withholding adjustments to an income-tax-only output."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model initially approximated the relevant deductions and obtained about $10,711, close to the correct rate computation, but then submitted $5,489 with no supporting calculation. That final number does not follow from its stated $72,247 taxable income or bracket arithmetic."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $4,500 of mortgage interest while omitting the $4,105.65 senior deduction and understating the engine-derived SALT deduction. These errors left taxable income at $77,683 instead of $72,807.33, and its unexplained adjustment from $11,802 to $11,210 did not repair the deduction calculation."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated partnership/S-corp income as self-employment income, deducted half of an invented self-employment tax, and chose a constructed standard-deduction total instead of itemizing. It then reduced an estimated $11,900 tax to $6,900 without identifying any nonrefundable credit."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the loan balance and omitted the additional senior deduction, leaving taxable income at $75,490 rather than $72,807.33. The correct itemized amount comes from deductible SALT, charity, and medical expenses without assuming an unlisted mortgage rate or payment."
-us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model limited SALT to the explicitly listed real-estate tax and omitted the $4,105.65 senior deduction. Consequently it used only $19,518 of itemized deductions rather than $24,787.23 and overstated taxable income as $82,183."
-us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $21,069.57 and omitted the $4,105.65 additional senior deduction. Those omissions raised taxable income to $80,630.83 instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly deducted half of self-employment tax from pass-through income, introduced a personal exemption, and applied obsolete 10%/15%/25% brackets. The applicable computation instead uses the $24,787.23 itemized deduction, QBI and senior deductions, and 2026 rates on $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used gross income of $110,094 even though only $19,828.80 of Social Security is taxable and AGI is $106,572.56. Its approximate $81,300 taxable income also omits part of the $33,765.23 total deductions."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a supposed 2026 reversion to pre-TCJA law, including personal exemptions, instead of the applicable 2026 deduction and rate regime. It therefore missed the QBI and additional senior deductions and taxed the income under the wrong parameters."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used sunset-era brackets and a personal exemption that is not part of this computation. It also understated itemized deductions and omitted the $4,105.65 senior deduction, so its $77,395 taxable income did not match $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $22,018.04 rather than $24,787.23 and omitted the $4,105.65 additional senior deduction. This inflated taxable income to $79,682.36."
-us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model chose the age-adjusted standard deduction, but itemization is beneficial once deductible SALT, charitable contributions, and medical expenses total $24,787.23. It also failed to reflect the separate $4,105.65 senior deduction in the stated derivation."
-us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly assumed the QBI deduction had expired and applied restored 10%/15%/25% brackets plus a personal exemption. For 2026 the $4,872.35 QBI deduction and $4,105.65 senior deduction apply, and tax is computed under the operative 2026 brackets."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the deductions as sufficient to eliminate income despite AGI of $106,572.56. After all $33,765.23 of deductions, taxable income remains $72,807.33 and produces positive federal income tax."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model excluded too much retirement and Social Security income and asserted that deductions eliminated the balance. Taxable pension and IRA distributions are fully included, $19,828.80 of Social Security is taxable, and taxable income remains $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a post-2025 personal exemption and regular brackets instead of the operative 2026 rules. It also counted only about $17,518 of itemized deductions and omitted the QBI and additional senior deductions that reduce taxable income to $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted half of self-employment tax from partnership/S-corp income and stacked an age-adjusted standard deduction with senior and QBI deductions. The correct choice is $24,787.23 of itemized deductions plus QBI and senior deductions, yielding $72,807.33 rather than about $67,020."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model chose the standard deduction and added a non-itemizer charitable deduction, thereby missing the superior $24,787.23 itemized deduction. With QBI and the senior deduction, taxable income is $72,807.33 rather than about $78,445."
-us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The answer implies that the model did not apply the full engine-derived deduction stack. Itemized deductions of $24,787.23, QBI of $4,872.35, and the $4,105.65 senior deduction reduce taxable income to $72,807.33 and tax to $10,729.61."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model supplied no usable derivation for its $10,234 figure. Applying the traced $33,765.23 total deductions to AGI gives $72,807.33 of taxable income, whose 2026 tax is $10,729.61 rather than the submitted amount."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied restored 10%/15%/25% brackets and a personal exemption, neither of which belongs in this 2026 calculation. It also omitted the QBI and additional senior deductions, leaving taxable income at $79,963 instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $22,264 and omitted the $4,105.65 additional senior deduction. These errors inflated taxable income to $79,437 instead of $72,807.33."
-us,scenario_114,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model incorrectly deducted half of self-employment tax from AGI, introduced a personal exemption, and used pre-TCJA brackets. It also failed to apply the separate QBI and senior deductions that form part of the $33,765.23 total reduction from AGI."
-us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so the required numeric result could not be parsed."
-us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model chose the $18,150 standard deduction after understating itemized deductions as $17,518, then added a $1,000 non-itemizer charitable deduction. The traced itemized deduction is $24,787.23 and, together with QBI and the senior deduction, yields $72,807.33 of taxable income."
-us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income and substantial itemized deductions but then asserted that unspecified nonrefundable credits reduced the tax to zero. No such credits apply, and $72,807.33 remains taxable after all deductions."
-us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted $19,828.80 of taxable Social Security from income and chose an understated standard deduction instead of itemizing. It also applied tax brackets inconsistently, since $15,654 does not follow from its stated $66,122 taxable income under the applicable 2026 rates."
-us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model miscomputed AGI as $80,164 despite listing income components that produce $106,572.56, understated QBI, and chose a standard deduction instead of the $24,787.23 itemized deduction. Its submitted $5,290.25 also does not follow from its own stated regular tax of $7,761.45 and claimed $491.50 QBI tax reduction."
+us,scenario_112,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated the ordinary 130% gross-income test as disqualifying and failed to apply categorical eligibility through TANF non-cash assistance. Its approximate net-income calculation also omitted SNAP-specific treatment that yields $1,269.19, after which the minimum allotment applies despite the expected contribution exceeding the maximum allotment."
+us,scenario_112,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model denied eligibility solely from a crude $20,843/12 gross-income calculation and never applied TANF non-cash categorical eligibility. The SNAP calculation produces qualifying gross and net income and then awards the minimum allotment."
+us,scenario_112,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,The model incorrectly made the ordinary 130% gross-income threshold dispositive and ignored categorical eligibility through TANF non-cash assistance. It also failed to carry the eligible household into the minimum-allotment step.
+us,scenario_112,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model excluded the $13,000 financial assistance from SNAP income and therefore understated net income, producing a large formula benefit. The engine derives $1,269.19 of net monthly income, for which the expected contribution exceeds the maximum allotment and only the minimum allotment is payable."
+us,scenario_112,snap,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $13,000 financial assistance from countable SNAP income and calculated net income from only wages and farm rent. Correct net income is $1,269.19 per month, so the household receives the minimum allotment rather than roughly $275 per month."
+us,scenario_112,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,The model applied the ordinary 130% gross-income test as an absolute bar and failed to apply TANF non-cash categorical eligibility. It consequently stopped before the minimum-allotment rule that pays an eligible one-person household even when its expected contribution exceeds the maximum allotment.
+us,scenario_112,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,The model treated its gross-income estimate as automatically disqualifying and ignored categorical eligibility through TANF non-cash assistance. The engine's SNAP-specific calculation instead yields qualifying gross and net income and an annual minimum allotment of $287.68.
+us,scenario_112,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model incorrectly ended the analysis at the ordinary 130% gross-income threshold. TANF non-cash categorical eligibility and the engine's $1,576.90 gross-income calculation preserve eligibility, after which the minimum allotment applies."
+us,scenario_112,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated SNAP net income as $1,440.20 instead of applying the deductions and income treatment that produce $1,269.19. That correct net income passes the net-income test, and categorical eligibility plus the minimum-allotment floor produces a positive benefit."
+us,scenario_112,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The unsupported zero implies that the model failed to recognize TANF non-cash categorical eligibility and the qualifying SNAP income tests. An eligible one-person household whose calculated benefit falls below zero receives the minimum allotment, not zero."
+us,scenario_112,snap,gemini-3.1-pro-preview,llm_error,other,False,"The model correctly identified that 30% of net income exceeds the maximum allotment but incorrectly converted that result to zero. SNAP applies the minimum-allotment floor to this eligible household, yielding $287.68 annually."
+us,scenario_112,snap,gemini-3.5-flash,llm_error,other,False,"The model both overstated net income and incorrectly stopped at a negative formula allotment. The engine derives qualifying net income of $1,269.19 and applies the minimum-allotment floor because the household remains eligible."
+us,scenario_112,snap,gemini-3.6-flash,llm_error,other,False,"The model incorrectly assumed that an expected contribution above the maximum allotment forces the benefit to zero. For this eligible household, that calculation triggers the minimum allotment, totaling $287.68 for the year."
+us,scenario_112,snap,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that household income exceeds the net-income limit. The SNAP-specific derivation gives net income of $1,269.19, or 97% of the poverty guideline, and eligibility leads to the minimum allotment."
+us,scenario_112,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model's simplified earned-income and standard deductions overstated net income at about $1,434. The engine's SNAP income treatment yields $1,269.19, which passes the net-income test and leads to the minimum allotment."
+us,scenario_112,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model's generic assertion missed TANF non-cash categorical eligibility and did not compute the qualifying gross and net income tests. It also failed to apply the minimum allotment once eligibility was established.
+us,scenario_112,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The prompt supplied household size, income, assets, and the assumptions needed for the SNAP calculation, so treating absent explicit eligibility indicators as zero was erroneous. The household qualifies through TANF non-cash categorical eligibility and receives the minimum allotment."
+us,scenario_112,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model treated the sum of annual cash-flow fields as SNAP gross income and applied the ordinary gross limit as dispositive. It missed TANF non-cash categorical eligibility and the engine's qualifying $1,576.90 monthly gross-income result."
+us,scenario_112,snap,gpt-5.6-luna,llm_error,categorical_eligibility,False,"The model incorrectly denied the household under the ordinary one-person gross-income limit. TANF non-cash categorical eligibility and SNAP-specific income treatment produce eligibility, followed by the minimum allotment."
+us,scenario_112,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model overstated net income by using only a simplified standard and earned-income deduction calculation. The engine derives $1,269.19 of net income, which passes the limit, and the household then receives the minimum allotment."
+us,scenario_112,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model incorrectly treated its gross-income sum as disqualifying and omitted the TANF non-cash categorical-eligibility pathway. The engine's SNAP gross income is $1,576.90 per month and passes the applicable test."
+us,scenario_112,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model gave a generic income-or-assets denial despite bank assets of only $10 and satisfied asset testing. It missed TANF non-cash categorical eligibility, qualifying income results, and the minimum-allotment floor."
+us,scenario_112,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model relied on approximate gross or net income without applying the engine's SNAP-specific calculation, which yields qualifying gross income of $1,576.90 and net income of $1,269.19. It therefore never reached the minimum-allotment step."
+us,scenario_112,snap,grok-4.6,llm_error,other,False,"The model incorrectly equated an expected contribution above the maximum allotment with a zero award. Because the household is eligible, SNAP substitutes the minimum allotment, producing $287.68 annually."
+us,scenario_112,snap,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied the 130% gross-income limit to a simple annual-income sum and treated it as conclusive. It missed TANF non-cash categorical eligibility and the SNAP-specific gross-income result of $1,576.90 per month."
+us,scenario_112,snap,inkling,llm_error,categorical_eligibility,False,The model incorrectly stopped at the ordinary 130% gross-income test. TANF non-cash categorical eligibility and the engine's qualifying income calculations lead to a minimum SNAP allotment instead of zero.
+us,scenario_112,snap,kimi-k2.6,llm_error,categorical_eligibility,False,The model treated 130% of poverty as the only relevant gross-income pathway for a nonelderly one-person household. It failed to apply TANF non-cash categorical eligibility and the qualifying SNAP-specific income calculation.
+us,scenario_112,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model explicitly assumed there was no TANF categorical eligibility, overlooking categorical eligibility through TANF non-cash assistance. It also treated an expected contribution above the maximum allotment as eliminating the award instead of triggering the minimum allotment."
+us,scenario_112,snap,minimax-m3,llm_error,categorical_eligibility,False,"The model inferred ineligibility merely from a one-person household having wage income and performed none of the required SNAP tests. The household satisfies the income and asset tests, qualifies categorically through TANF non-cash assistance, and receives the minimum allotment."
+us,scenario_112,snap,ox-alpha,llm_error,categorical_eligibility,False,"The model used a direct sum of wages, assistance, and farm rent and treated the ordinary 130% gross limit as dispositive. It missed TANF non-cash categorical eligibility and the engine's lower SNAP gross-income result of $1,576.90 per month."
+us,scenario_112,snap,qwen-3.7-max,llm_error,other,False,"The model's submitted $3,576 contradicts its own calculation that countable income makes the ordinary formula nonpositive, and it invented an unlisted shelter deduction despite the instruction to set unlisted expenses to zero. The correct next step is the minimum-allotment floor for the eligible household, not an amount near the maximum allotment."
+us,scenario_112,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The bare ineligibility conclusion missed TANF non-cash categorical eligibility and the engine's passing gross, net, asset, work, and immigration tests. Once eligibility is established, the minimum allotment produces a positive annual benefit."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions by counting only $4,064 of SALT instead of the modeled $9,431.59 and by deriving only about $4,007 of medical deductions instead of $4,441.86. It then abandoned its own approximately $11,889 bracket calculation and asserted $8,971 without a supporting tax computation."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model omitted taxable Social Security from AGI, chose a fabricated $28,300 standard deduction instead of the larger $24,787.23 itemized deduction, and incorrectly described QBI as reducing AGI. It also added employee Medicare tax and withholding adjustments to an income-tax output, neither of which belongs in federal income tax before refundable credits."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used the wrong itemized components, including an unsupported $10,000 SALT amount and only $10,000 of medical expenses, then applied the senior deduction at an unphased $6,000 instead of $4,105.65. After computing tax near $10,711, it submitted $5,489 with no arithmetic producing that amount."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented roughly $4,500 of mortgage interest from the mortgage balance, omitted the $4,105.65 senior deduction, and failed to use the modeled $9,431.59 SALT deduction. Those errors left taxable income at $77,683 rather than $72,807.33, and its final $11,210 also does not follow from its stated $11,802 calculation."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,payroll_tax_base,False,"The model treated partnership/S-corporation income as self-employment income, deducted an unsupported half of self-employment tax from AGI, and then chose the standard deduction instead of the $24,787.23 itemized deduction. It further reduced an approximately $11,900 bracket tax to $6,900 without identifying any nonrefundable credit."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented $6,693 of mortgage interest and omitted both the modeled state-income-tax portion of the $9,431.59 SALT deduction and the $4,105.65 senior deduction. Consequently it used $75,490 of taxable income rather than $72,807.33."
+us,scenario_114,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model limited SALT to the listed $4,064 real-estate tax, omitting modeled Virginia income tax, and omitted the $4,105.65 additional senior deduction. Its $19,518 itemized total therefore fell well below the correct $24,787.23, inflating taxable income to $82,183."
+us,scenario_114,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model understated medical deductions at $2,007.04 by excluding the $2,000 over-the-counter health expense and understated SALT at $7,615.53. It also omitted the $4,105.65 senior deduction, producing $80,630.83 rather than $72,807.33 of taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model incorrectly deducted half of self-employment tax from partnership/S-corporation income, used a personal exemption, and applied obsolete 10%/15%/25% brackets. It also failed to use the actual $24,787.23 itemized, $4,872.35 QBI, and $4,105.65 senior deductions that produce $72,807.33 of taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used $110,094 of gross income even though 85%-taxable Social Security produces $106,572.56 of AGI, and its approximate $81,300 taxable income omits substantial deductions. The full itemized, QBI, and senior deductions reduce taxable income to $72,807.33."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model applied a pre-TCJA personal-exemption regime instead of the governing 2026 provisions. It failed to apply the actual deduction package of $24,787.23 itemized, $4,872.35 QBI, and $4,105.65 senior deductions."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model used an inapplicable $5,050 personal exemption and sunset-era tax brackets while understating itemized deductions and omitting the $4,105.65 senior deduction. The governing deductions produce $72,807.33 of taxable income taxed under the 2026 10%/12%/22% schedule."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $22,018.04 instead of $24,787.23 and omitted the $4,105.65 additional senior deduction. This left taxable income $6,875.03 too high."
+us,scenario_114,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model selected the standard deduction even though $24,787.23 of itemized deductions exceeds the $18,150 standard deduction. It also failed to account for the full $4,105.65 senior deduction in the derivation implied by its $13,104.09 tax."
+us,scenario_114,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model incorrectly treated the QBI deduction as expired and applied a personal exemption and restored 10%/15%/25% brackets. The 2026 calculation instead includes a $4,872.35 QBI deduction and $4,105.65 senior deduction and uses the continuing 10%/12%/22% brackets at this income."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model treated the deductions as sufficient to erase income despite $106,572.56 of AGI. The allowed $33,765.23 deduction package leaves $72,807.33 taxable, not zero."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model broadly excluded retirement and Social Security income even though the facts produce $30,000 of taxable pension, $9,600 of taxable IRA distributions, and $19,828.80 of taxable Social Security. After all deductions, $72,807.33 remains taxable."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used a post-2025 personal exemption and understated itemized deductions at about $17,518 while omitting the QBI and senior-deduction treatment reflected in the governing law. These errors raised taxable income from $72,807.33 to about $83,700 and invoked the wrong bracket regime."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,payroll_tax_base,False,"The model improperly deducted half of estimated self-employment tax from partnership/S-corporation income and then overstated the combined deduction effect, reducing taxable income to about $67,020. The trace contains only $22 of above-the-line deductions and yields $72,807.33 of taxable income."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model chose standard-deduction treatment and added a non-itemizer charitable deduction, missing that $24,787.23 of itemized deductions is superior. With the $4,872.35 QBI and $4,105.65 senior deductions, taxable income is $72,807.33 rather than about $78,445."
+us,scenario_114,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted tax implies the model did not apply the traced deduction amounts in full. The exact $24,787.23 itemized deduction, $4,872.35 QBI deduction, and $4,105.65 senior deduction produce $72,807.33 of taxable income and $10,729.61 of tax."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model supplied no computation supporting $10,234. Applying the traced $33,765.23 total deduction package to $106,572.56 of AGI yields $72,807.33 taxable income, whose 2026 bracket tax is $10,729.61."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied an obsolete personal exemption and restored 10%/15%/25% brackets. It also omitted the $4,872.35 QBI and $4,105.65 senior deductions and understated itemized deductions, so its $79,963 taxable income is not the governing calculation."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model wrongly eliminated QBI, added a personal exemption, and used restored 10%/15%/25% brackets. The actual 2026 calculation uses $24,787.23 of itemized deductions, a $4,872.35 QBI deduction, a $4,105.65 senior deduction, and the continuing lower brackets."
+us,scenario_114,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $22,264, chiefly through an understated $6,810 SALT amount, and omitted the $4,105.65 senior deduction. Those omissions raised taxable income to $79,437 instead of $72,807.33."
+us,scenario_114,federal_income_tax_before_refundable_credits,inkling,llm_error,thresholds_rates,False,"The model improperly deducted half of self-employment tax from AGI, substituted a personal exemption for the applicable QBI and senior deductions, and applied pre-TCJA brackets. Partnership/S-corporation income does not support its self-employment-tax adjustment, and the traced AGI is $106,572.56."
+us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model returned no value or explanation for the requested output.
+us,scenario_114,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model incorrectly selected the $18,150 standard deduction because its $17,518 itemized estimate omitted modeled Virginia income tax from SALT and understated deductible medical expenses. The correct $24,787.23 itemized deduction is larger, and no separate $1,000 non-itemizer charitable deduction is added when itemizing."
+us,scenario_114,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income and then asserted that unspecified nonrefundable credits reduced the liability to zero, although no such credits apply. The allowed deductions leave $72,807.33 taxable and therefore a positive $10,729.61 regular income tax."
+us,scenario_114,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model chose the standard deduction because it understated itemized deductions at about $19,500, omitting modeled Virginia income tax from SALT and understating medical deductions. The correct $24,787.23 itemized deduction exceeds the standard deduction and lowers taxable income to $72,807.33."
+us,scenario_114,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted $19,828.80 of taxable Social Security from income, used incorrect standard and age-addition amounts, and failed to itemize. It then produced $15,654 of tax from its stated $66,122 taxable income using no valid 2026 bracket arithmetic."
+us,scenario_114,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model misadded income as $80,164 despite listing components that produce $106,572.80, incorrectly altered the $22 capital loss, and understated QBI. It also chose a $20,400 standard deduction rather than the $24,787.23 itemized deduction, so neither its taxable income nor its $5,290.25 tax follows from the facts."
 us,scenario_114,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model applied the childless EITC phaseout using only $22,700 of wages and ignored the rule requiring the greater of earned income or adjusted gross income. The filer’s other taxable income raises AGI beyond the childless EITC phaseout ceiling, yielding EITC of $0 rather than $295."
 us,scenario_114,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so it failed the required submission contract."
 us,scenario_114,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The model correctly calculated both employee payroll-tax components and correctly excluded self-employment tax, but submitted $3,470.30 instead of their $1,736.55 sum. Its answer effectively duplicates the employee payroll-tax total despite its own reasoning identifying no additional component."
@@ -6607,36 +6930,38 @@ us,scenario_114,self_employment_tax,inkling,llm_error,payroll_tax_base,False,"Th
 us,scenario_114,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no self_employment_tax value or explanation, violating the required output contract."
 us,scenario_114,self_employment_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly treated the $24,362 partnership or S-corporation income as self-employment income and computed tax on 92.35% of it. Its submitted $3,459.40 also contradicts its own stated calculation of $3,442.62; the correct base for this output is zero."
 us,scenario_114,self_employment_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model incorrectly classified the $24,362 partnership or S-corporation income as self-employment earnings. It also added a purported amount above the Social Security wage base even though wages plus its assumed earnings were far below that base; with zero self-employment earnings, neither component applies."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model's own taxable-income estimate near $66,296 would produce roughly $3,555 under the brackets it stated, yet it submitted $2,597 without a supporting adjustment. It also overstated itemized deductions instead of using $19,419.94 and omitted the $800 aged exemption from the $1,730 exemption total."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted charitable contributions and employee expenses in computing federal AGI, omitted pension and IRA income from its stated result, and used an inapplicable $28,000 deduction. Virginia taxable income is $65,593.83, not an amount sheltered to zero."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used the Virginia standard deduction instead of the larger $19,419.94 itemized deduction and omitted the $1,730 exemption total from taxable income. Its narrative computed about $4,227 but then submitted $1,854 with no valid intervening computation."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model never completed the itemized-deduction calculation and retained a result close to its standard-deduction method. The required deductions are $19,419.94 of itemized deductions plus $1,730 of exemptions, producing taxable income of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of the larger $19,419.94 itemized deduction and did not correctly subtract the $930 personal and $800 aged exemptions from income. It also described exemptions as credits rather than deductions in determining Virginia taxable income."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction despite itemized deductions of $19,419.94 being larger. It then subtracted the $930 personal exemption and $800 aged exemption dollar-for-dollar from tax, whereas they reduce taxable income, yielding $65,593.83 before applying the brackets."
-us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated Social Security as federally excluded and allowed a $12,000 Virginia age deduction even though that deduction is fully phased out at this income. It also omitted the $1,730 personal and aged exemptions and understated itemized deductions relative to the traced $19,419.94."
-us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model allowed a $12,000 age deduction that is fully phased out and used an $8,500 standard deduction instead of $19,419.94 of itemized deductions. It also omitted the $1,730 exemption total, so its taxable-income base does not match $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated itemized deductions at $23,948.60 and subtracted only an $800 exemption. The traced amounts are $19,419.94 of itemized deductions and $1,730 of exemptions, yielding taxable income of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,180 implies taxable income around $77,174, consistent with using a standard deduction and incomplete exemptions. The correct computation subtracts $19,419.94 of itemized deductions and $1,730 of exemptions from Virginia AGI, leaving $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Although the model named the Social Security subtraction and itemized deductions, its $3,726 answer implies taxable income near $69,278 rather than $65,593.83. It failed to use the full traced $19,419.94 itemized deduction and $1,730 exemption total."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions as $15,511 instead of $19,419.94 and misstated exemptions as $1,600 instead of $1,730. Those errors inflated taxable income from $65,593.83 to its stated $69,633."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model used only $17,518.04 of itemized deductions rather than $19,419.94. With the correctly stated $1,730 exemptions, that $1,901.90 deduction shortfall explains its inflated taxable income and tax."
-us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $3,623.52 answer corresponds to taxable income of $67,495.96, the result of using only $17,518.04 in itemized deductions. The full itemized deduction is $19,419.94, which lowers taxable income to $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model's $14,036.62 cannot result from applying Virginia's stated brackets to any taxable-income figure derived in its narrative; even its standard-deduction approach produces tax near $4,200. It also invoked a nonexistent available pension deduction instead of using $19,419.94 of itemized deductions and $1,730 of exemptions."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared all income sheltered by deductions and exemptions. After the Social Security subtraction, itemized deductions, and exemptions, Virginia taxable income remains $65,593.83 and generates $3,514.15 of tax."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated retirement income and age-based relief as sufficient to eliminate taxable income. Taxable IRA distributions and pension income remain in Virginia AGI, and the allowed deductions and exemptions leave $65,593.83 taxable."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $17,518 rather than $19,419.94. Its Social Security subtraction and $1,730 exemptions were otherwise aligned, so the missing $1,901.94 deduction directly inflated taxable income and tax."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model allowed an age deduction that is fully phased out, driving taxable income down to about $51,647. The applicable age-related amount is the $800 aged exemption within total exemptions of $1,730, not a $12,000 subtraction."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used an $8,750 standard deduction and only the $930 personal exemption. It needed to itemize $19,419.94 and also apply the $800 aged exemption, producing taxable income of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $4,658 answer implies taxable income around $85,487, showing that most of the available deductions and exemptions were not applied. Virginia AGI must be reduced by $19,419.94 of itemized deductions and $1,730 of exemptions before applying the brackets."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction and exemptions eliminate Virginia taxable income. The applicable itemized deductions and exemptions leave $65,593.83 taxable, not zero."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions at $17,518 instead of $19,419.94. This raised its taxable income to $67,496 rather than the traced $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model wrongly allowed a fully phased-out $12,000 age deduction and used a 5.3% top rate instead of Virginia's 5.75% rate. It also omitted the $1,730 exemptions and slightly overstated itemized deductions, so both its tax base and rate computation were wrong."
-us,scenario_114,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used only the $930 personal exemption and omitted the $800 aged exemption. It also used approximately $19,647 of itemized deductions rather than $19,419.94; together these errors produced taxable income of about $64,446 instead of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured numeric answer was missing."
-us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $19,419.94 of itemized deductions and misstated the aged exemption as $930 rather than $800. The correct exemption total is $1,730, producing taxable income of $65,593.83."
-us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model labeled the filer low-income and treated deductions and an age exemption as eliminating liability. Virginia taxable income remains $65,593.83 after all applicable subtractions, itemized deductions, and exemptions."
-us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented an available $12,000 pension or age deduction, subtracted the full Social Security benefit after already excluding it from its Virginia income base, and used an obsolete $1,000 standard deduction. It also omitted the $19,419.94 itemized deduction and $1,730 exemptions that actually determine taxable income."
-us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model used an incorrect federal AGI, allowed a fully phased-out $12,000 age deduction, and used a standard deduction instead of $19,419.94 of itemized deductions. It then invented $745.50 of nonrefundable credits; the $930 personal and $800 aged amounts are exemptions deducted from income, not tax credits."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The model’s own taxable-income calculation of roughly $66,296 would produce about $3,555 under the brackets it stated, yet it submitted $2,597 without any valid adjustment. It also overstated itemized deductions at about $19,518 instead of $19,419.94 and omitted the $800 aged exemption."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model improperly deducted charitable gifts and employee expenses in computing federal AGI, omitted taxable Social Security, and then applied an inapplicable $28,000 standard deduction. The correct Virginia taxable-income derivation produces $65,593.83, not zero."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,other,False,"The model reasoned to approximately $4,227 using the standard deduction but submitted $1,854 with no supporting computation. It also failed to use the larger $19,419.94 itemized deduction and the full $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model never completed the itemized-deduction calculation and instead estimated an unsupported deduction near $17,000. Virginia itemized deductions are $19,419.94 and exemptions are $1,730, producing taxable income of $65,593.83 and tax of $3,514.15."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used the Virginia standard deduction rather than the larger $19,419.94 itemized deduction. It also described exemptions as credits instead of subtracting the $930 personal and $800 aged exemptions from Virginia AGI."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction instead of $19,419.94 of itemized deductions, then incorrectly subtracted the $1,730 of exemptions dollar-for-dollar from tax. Virginia exemptions reduce taxable income, yielding $65,593.83 before the tax brackets are applied."
+us,scenario_114,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated Social Security as fully excluded from federal AGI and granted a $12,000 Virginia age deduction despite the income phaseout. It also omitted the $1,730 of Virginia exemptions and understated itemized deductions, so its $55,739 taxable-income estimate was invalid."
+us,scenario_114,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model granted a $12,000 age deduction that is fully eliminated by the income limitation, then used the standard deduction and omitted the $1,730 of exemptions. The applicable reductions are the Social Security subtraction, $19,419.94 of itemized deductions, and $1,730 of exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated itemized deductions at $23,948.60 and allowed only an $800 exemption. The trace uses $19,419.94 of itemized deductions plus both the $930 personal and $800 aged exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $4,180 implies taxable income near $77,174, consistent with using a standard deduction rather than the household’s larger itemized deductions. Correctly subtracting $19,419.94 of itemized deductions and $1,730 of exemptions leaves $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Although the model named the Social Security subtraction and itemized deductions, its $3,726 answer implies taxable income near $69,278. It failed to apply the full $19,419.94 itemized deduction and $1,730 exemption total that reduce taxable income to $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions as $15,511 instead of $19,419.94 and misstated exemptions as $1,600 instead of $1,730. Those errors overstated Virginia taxable income by $4,039.17."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model understated Virginia itemized deductions by $1,901.90, using $17,518.04 rather than $19,419.94. This overstated taxable income by the same amount and produced $3,623.52 instead of $3,514.15."
+us,scenario_114,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The $3,623.52 result corresponds to taxable income of $67,495.96, which uses only $17,518.04 of itemized deductions after the Social Security subtraction and exemptions. The correct itemized deduction is $19,419.94, leaving taxable income of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,glm-5.2,llm_error,other,False,"The model’s described Virginia brackets applied to the income components it listed cannot produce $14,036.62; that amount exceeds the tax on the entire stated federal AGI. It also invented a pension deduction and used the standard deduction instead of the traced $19,419.94 itemized deduction."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that deductions and exemptions shelter all income. After the Social Security subtraction, itemized deductions, and exemptions, Virginia taxable income remains $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated the filer’s retirement income and age-based reductions as sufficient to eliminate taxable income. Virginia still taxes the IRA and private pension income, and the valid reductions leave $65,593.83 taxable."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions as $17,518 rather than $19,419.94. This inflated Virginia taxable income from $65,593.83 to about $67,496 and consequently overstated tax."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model applied an age deduction that is eliminated under Virginia’s income limitation. The valid Social Security subtraction, $19,419.94 itemized deduction, and $1,730 exemptions produce taxable income of $65,593.83, not approximately $51,647."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model selected an $8,750 standard deduction rather than the larger $19,419.94 itemized deduction and omitted the $800 aged exemption. Those choices overstated taxable income by about $11,470."
+us,scenario_114,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $4,658 answer implies Virginia taxable income near $85,496, so the model failed to apply most of the traced reductions. The Social Security subtraction, itemized deductions, and exemptions reduce the tax base to $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that the standard deduction and exemptions offset all taxable income. Even after the larger itemized deduction is used, Virginia taxable income is $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated itemized deductions as $17,518 instead of $19,419.94. Its remaining steps were aligned, but the $1,901.94 deduction shortfall inflated taxable income and tax."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used $19,518 of itemized deductions instead of $19,419.94, understating taxable income by about $98. It therefore calculated tax on roughly $65,496 rather than the traced $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model both granted a fully phased-out $12,000 age deduction and used a nonexistent 5.3% Virginia top rate instead of 5.75%. These errors drove taxable income and the tax rate below the applicable values."
+us,scenario_114,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model allowed only the $930 personal exemption and omitted the $800 aged exemption while also misstating itemized deductions as about $19,647. The correct deductions and $1,730 exemption total leave taxable income of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the response failed the required output contract."
+us,scenario_114,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction rather than $19,419.94 of itemized deductions and misstated the aged exemption as $930 rather than $800. These errors produced taxable income of $76,384 instead of $65,593.83."
+us,scenario_114,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model mislabeled the filer as low-income and treated deductions and an age exemption as eliminating liability. The correct reductions still leave $65,593.83 of Virginia taxable income."
+us,scenario_114,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model failed to subtract Virginia’s $19,828.80 Social Security subtraction and used a $9,500 standard deduction rather than $19,419.94 of itemized deductions. It also omitted the applicable $930 personal and $800 aged exemptions."
+us,scenario_114,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model invented a $12,000 pension deduction, subtracted the full $23,328 Social Security benefit after already excluding it from its Virginia income base, and used an obsolete $1,000 standard deduction. It also introduced an unsupported half-self-employment-tax deduction and did not submit the $2,542.44 result produced by its own arithmetic."
+us,scenario_114,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used an incorrect $80,164 federal AGI, granted a fully phased-out $12,000 age deduction, and used the standard deduction instead of $19,419.94 of itemized deductions. It then invented $745.50 of nonrefundable credits, none of which appears in the traced calculation."
 us,scenario_114,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_115,head_medicaid_eligible,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model invented an Alabama aged/disabled Medicaid pathway for the head instead of determining whether PolicyEngine assigned an eligible category. The engine assigns medicaid_category = NONE, so comparing $18,708 of Social Security income with an asserted aged/disabled limit cannot produce eligibility."
 us,scenario_115,head_medicaid_eligible,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model treated age, disability, low assets, and $1,953 of monthly income as sufficient for an Alabama aged/disabled Medicaid category. PolicyEngine assigns no Medicaid category, so its income-and-assets comparison was applied to a pathway the head does not qualify through."
@@ -6649,35 +6974,37 @@ us,scenario_115,snap,claude-opus-4.8,llm_error,thresholds_rates,False,"The model
 us,scenario_115,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model waived the gross-income test, excluded tax-exempt interest, and converted a negative allotment formula into an unsupported $142 monthly benefit. Total income of approximately $23,442 exceeds the applicable eligibility limit, and minimum-benefit rules cannot override that ineligibility."
 us,scenario_115,snap,claude-sonnet-5,llm_error,thresholds_rates,False,"The model correctly computed that the ordinary allotment formula yields $0 even after excluding tax-exempt interest, but then replaced that result with an unsupported $321 monthly minimum benefit. It also missed that approximately $23,442 of income exceeds the applicable SNAP eligibility threshold; minimum-allotment rules do not grant benefits after failure of eligibility."
 us,scenario_115,snap,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced all countable income to zero through a standard medical deduction. The listed $250 of annual medical and over-the-counter expenses cannot erase approximately $23,442 of annual household income, which exceeds the applicable one-person SNAP eligibility limit."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model excluded the entire $4,734 interest amount from Alabama AGI. Alabama includes that interest here, then subtracts $4,500 of deductions and exemptions, leaving $234 taxable at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model excluded the $4,734 interest amount and invoked unspecified elderly exemptions and nonrefundable credits to eliminate liability. The applicable deductions and exemption total $4,500, leaving $234 taxable and $4.68 due."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $4,734 interest amount as excluded from Alabama taxable income. It enters Alabama AGI here, leaving $234 after the $4,500 deduction and exemption total."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model set Alabama AGI to zero by excluding both Social Security and the $4,734 interest amount. Alabama excludes the Social Security but includes the interest, producing $234 of taxable income after deductions."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest amount from Alabama AGI. After the $3,000 standard deduction and $1,500 personal exemption, $234 remains taxable at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly excluded Social Security but incorrectly excluded the $4,734 interest amount from Alabama gross income. Including it and subtracting $4,500 leaves $234 taxable, yielding $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model treated both listed income sources as exempt from Alabama income tax. The $4,734 interest amount is included in Alabama AGI here, and $234 remains after deductions and exemptions."
-us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model excluded the $4,734 interest amount and therefore set state taxable income to zero. Alabama includes that amount here and taxes the $234 remaining after the $4,500 deduction and exemption total."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated the deductions and exemptions as at least $5,000 based on the filer's age. The applicable standard deduction and personal exemption total $4,500, so $234 of the $4,734 Alabama AGI remains taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that taxable income fell below the threshold for liability. Alabama AGI is $4,734, deductions and exemptions total $4,500, and the remaining $234 is taxed at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model excluded Social Security and then incorrectly concluded that no taxable income remained, overlooking the $4,734 interest included in Alabama AGI. That amount leaves $234 taxable after the applicable deductions."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest amount from Alabama taxable income. Alabama includes it here, and the $4,500 deduction and exemption total leaves $234 taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model treated the Social Security exemption as eliminating all Alabama taxable income. It omitted the $4,734 interest included in Alabama AGI, of which $234 remains taxable after deductions."
-us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model stated that there was no income other than exempt Social Security, overlooking the $4,734 interest included in Alabama AGI. After $4,500 of deductions and exemptions, $234 remains taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $4,734 interest amount as federally tax-exempt interest that Alabama also excludes. Alabama includes it in AGI here, leaving $234 taxable after the $4,500 deduction and exemption total."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model classified the household as having no Alabama-taxable income because it excluded the $4,734 interest amount. Alabama includes that amount here and taxes the $234 remaining after deductions."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced taxable income to zero under its simplified deduction calculation. The applicable deductions and exemption total $4,500 against $4,734 of Alabama AGI, leaving $234 rather than zero."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that no Alabama-taxable income remained after deductions. The $4,734 interest amount enters Alabama AGI, and the applicable $4,500 total deduction and exemption leave $234 taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly used the absence of federal taxable income to set Alabama taxable income to zero. Alabama includes the $4,734 interest amount in its AGI calculation here, leaving $234 taxable after deductions."
-us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the reported interest produced no Alabama taxable income. Alabama includes the $4,734 interest amount here, and $234 remains after the $4,500 deduction and exemption total."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model stated that no taxable income existed beyond exempt Social Security, omitting the $4,734 interest included in Alabama AGI. The resulting $234 of taxable income produces $4.68 of tax."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model correctly considered adding back the $4,734 interest but incorrectly assumed that Alabama deductions and exemptions fully offset it. They total $4,500, leaving $234 taxable at 2%."
-us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed that deductions and exemptions were sufficient to eliminate the $4,734 interest amount if included. The applicable total is $4,500, leaving $234 taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model recognized the Alabama addition of the $4,734 interest but incorrectly concluded that the standard deduction and personal exemption reduced it to zero. Those amounts total $4,500, so $234 remains taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest amount from Alabama income along with Social Security. Alabama includes the interest here, leaving $234 taxable after the $4,500 deduction and exemption total."
-us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model overstated the combined standard deduction and personal exemption as at least $5,500. The applicable amounts are $3,000 and $1,500, totaling $4,500 and leaving $234 taxable."
-us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Alabama has no state income tax. Alabama imposes an individual income tax, and the first $234 of taxable income here is taxed at 2%, producing $4.68."
-us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest amount from Alabama taxable income. Alabama includes that amount here, while excluding Social Security, and $234 remains taxable after deductions."
-us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the $4,734 Alabama AGI to zero with the standard deduction. The standard deduction and personal exemption total $4,500, leaving $234 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the entire $4,734 interest amount from Alabama AGI. After the $3,000 standard deduction and $1,500 personal exemption, $234 remains taxable at 2%, producing $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the interest as excluded and invoked unspecified elderly exemptions and nonrefundable credits that do not erase the liability. Alabama AGI is $4,734, and the applicable $4,500 combined standard deduction and personal exemption leave $234 taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama taxable income. Social Security is exempt, but the interest forms Alabama AGI and leaves $234 after the applicable deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model incorrectly set Alabama AGI to zero by excluding the $4,734 interest. The interest forms Alabama AGI, from which $4,500 of deductions leaves $234 taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $4,734 interest as absent from Alabama AGI. Applying the $3,000 standard deduction and $1,500 personal exemption leaves $234 taxable rather than zero."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model correctly exempted Social Security but incorrectly excluded the $4,734 interest from Alabama gross income. That interest yields $234 of taxable income after the applicable $4,500 of deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that neither listed income source enters Alabama taxation. Although Social Security is exempt, the $4,734 interest enters Alabama AGI and leaves $234 taxable after deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest and therefore set state taxable income to zero. Alabama's applicable deductions total $4,500, leaving $234 subject to the 2% rate."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model overstated the deductions and exemptions as at least $5,000. The applicable standard deduction and personal exemption total $4,500, so $234 of the $4,734 AGI remains taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that taxable income falls below the liability threshold. The $4,734 Alabama AGI exceeds the applicable $4,500 of deductions by $234, which is taxed at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model treated the Social Security exemption as eliminating all Alabama taxable income and omitted the $4,734 interest from AGI. That interest leaves $234 taxable after deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama AGI. With $4,500 in applicable deductions, the correct taxable-income remainder is $234."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model focused only on Alabama's Social Security exemption and omitted the $4,734 interest that forms state AGI. After deductions, $234 remains subject to Alabama income tax."
+us,scenario_115,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly said there was no other taxable income after exempting Social Security. The $4,734 interest enters Alabama AGI and leaves $234 taxable after the standard deduction and personal exemption."
+us,scenario_115,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the $4,734 interest as federally tax-exempt interest excluded from Alabama AGI. PolicyEngine includes it in Alabama AGI, leaving $234 taxable after $4,500 of deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the listed interest was not Alabama-taxable. The $4,734 interest forms state AGI and exceeds the applicable deductions by $234."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced taxable income to zero after deductions. The applicable deductions total $4,500 against $4,734 of Alabama AGI, leaving $234 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly stated that no Alabama-taxable income remained after deductions. The $4,734 interest enters Alabama AGI, and the $4,500 standard deduction plus personal exemption leave $234 taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model improperly equated the absence of federal taxable income with zero Alabama taxable income and excluded the interest. Alabama AGI includes the $4,734 interest, leaving $234 after deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model incorrectly said the reported interest produced no Alabama taxable income. It forms $4,734 of Alabama AGI, of which $234 remains after the applicable deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model correctly exempted Social Security but incorrectly asserted that no other taxable income existed. The $4,734 interest enters Alabama AGI and leaves $234 taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model overstated the deductions available against the $4,734 interest. The applicable standard deduction and personal exemption total $4,500, leaving $234 rather than zero."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly limited Alabama-taxable income to wages and business income and omitted the $4,734 interest from state AGI. After $4,500 of deductions, $234 remains taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed the deductions were sufficient to offset all $4,734 of interest. The applicable deductions total only $4,500, leaving $234 taxable at 2%."
+us,scenario_115,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model recognized that Alabama adds the $4,734 interest but incorrectly claimed the standard deduction and personal exemption fully offset it. Those deductions total $4,500, leaving $234 taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama taxable income along with the exempt Social Security benefits. The interest forms Alabama AGI and leaves $234 taxable after deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model overstated the low-income standard deduction plus personal exemption as at least $5,500. The applicable amounts are $3,000 and $1,500, totaling $4,500 and leaving $234 taxable."
+us,scenario_115,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The model falsely stated that Alabama has no state income tax. Alabama imposes an individual income tax, and the household's $234 of taxable income is taxed at 2%, producing $4.68."
+us,scenario_115,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model incorrectly excluded the $4,734 interest from Alabama AGI. That amount is the household's state AGI and leaves $234 taxable after the $4,500 of applicable deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model correctly identified the Social Security exemption but incorrectly excluded the $4,734 interest from Alabama taxable income. The interest enters state AGI and leaves $234 taxable after deductions."
+us,scenario_115,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly concluded that the standard deduction reduced Alabama taxable income to zero. The $3,000 standard deduction plus $1,500 personal exemption leave $234 of the $4,734 AGI taxable."
 us,scenario_115,tanf,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_116,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model improperly awarded a SNAP minimum allotment after its own benefit calculation reduced the regular allotment below zero, even though the household fails the gross-income eligibility test. It also submitted $6,804 despite stating an annual result of $3,402, doubling its own incorrectly calculated amount."
 us,scenario_116,spouse_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model invented expanded or special disabled-adult coverage and treated low assets and potentially favorable treatment of veterans benefits as sufficient. Florida assigns the spouse no Medicaid category, so neither the income comparison nor the $4,870 resource amount creates eligibility."
@@ -6691,36 +7018,38 @@ us,scenario_117,child3_early_head_start_eligible,gpt-5.4-mini,llm_error,categori
 us,scenario_117,child3_early_head_start_eligible,kimi-k2.6,llm_error,categorical_eligibility,False,"The model treated remaining under age 3 throughout the year as sufficient to qualify as a participant. It omitted the required low-income condition, and this household's approximately $250,733 income is above the qualifying threshold."
 us,scenario_117,child3_head_start_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model treated age 2 as sufficient for Head Start eligibility and omitted the income-eligibility test. The household's approximately $250,733 income exceeds the poverty-based Head Start limit, and no other qualifying pathway applies."
 us,scenario_117,child3_wic_eligible,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model treated age under five as sufficient for WIC eligibility and omitted the separate income test. Although Child 3 meets the categorical age rule, household income exceeds the 185% federal-poverty-guideline threshold for a five-person household, yielding no WIC eligibility."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"It incorrectly phased the $25,000 qualified-overtime deduction down to $15,185; the trace applies the full capped $25,000 deduction. It also subtracted a $1,200 CDCC even though the head has no earned income and omitted the $2,000 charitable deduction allowed alongside the standard deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It double-counted the $35,525 overtime premium on top of wages that already include overtime, included the state refund, and then treated charity and employee expenses as AGI deductions. It also omitted the $25,000 overtime deduction, used the wrong standard deduction, and failed to subtract the $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It invented mortgage interest from the mortgage balance and an assumed rate, invented a $10,000 SALT deduction, and included the state refund. It also used the wrong child-credit amounts and subtracted a CDCC despite the head having no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its stated calculation reaches approximately $25,972, but it submitted $47,714, so the final value does not follow its own arithmetic. The underlying work also invents mortgage interest and SALT deductions, includes the state refund, omits the $25,000 overtime deduction, and subtracts an unavailable CDCC."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"It fabricated roughly $80,000 of itemized deductions from an assumed mortgage rate and unlisted SALT, then stacked the overtime deduction on top of them. The trace instead uses the $32,200 standard deduction, $25,000 overtime deduction, and $2,000 charitable deduction, with no CDCC."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It included the $2,589 state refund, used a $30,000 standard deduction, and omitted both the $25,000 overtime deduction and $2,000 charitable deduction. It also subtracted a $1,200 CDCC even though the head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its own detailed computation produces $22,630, but it submitted $62,700, an unexplained arithmetic-output mismatch. Its computation also invents mortgage interest and SALT, includes the state refund, omits the overtime deduction, understates the CTC, and subtracts an unavailable CDCC."
-us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"It applied obsolete 15%, 25%, and 28% pre-TCJA brackets instead of the applicable 2026 rate schedule. It also invented SALT and medical deductions, included the state refund, omitted the overtime and charitable deductions used by the trace, and wrongly denied the unphased $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"It used pre-TCJA rates and personal exemptions rather than the applicable 2026 schedule and deductions. It also invented SALT and medical deductions, included the state refund, understated the CTC, and subtracted a CDCC barred by the head's lack of earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"Its $46,522 answer does not reflect the traced taxable income of $188,944.53 after the $32,200 standard deduction, $25,000 overtime deduction, and $2,000 charitable deduction. It also fails to reflect the resulting $30,991.80 bracket tax less the full $6,600 CTC."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"It explicitly used pre-TCJA 2026 rules instead of the applicable 2026 rate and deduction regime. That choice discards the traced $32,200 standard deduction, $25,000 overtime deduction, $2,000 charitable deduction, and $6,600 CTC computation."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"It wrongly applied post-expiration pre-TCJA rules and treated the child credit as $1,000 per child fully phased out at this income. The applicable CTC is $2,200 for each of three children and is not phased out, while the traced deductions produce taxable income of $188,944.53."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Its result does not implement the traced deduction stack: $32,200 standard deduction, full $25,000 overtime deduction, and $2,000 charitable deduction. It also invokes a high-income CTC phaseout even though the full $6,600 credit applies."
-us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"Its $41,320 answer fails to reduce AGI by the traced $59,200 total deductions and then subtract the $6,600 CTC. Those steps produce taxable income of $188,944.53 and tax before refundable credits of $24,391.80."
-us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It included the state refund and deducted only an approximate standard deduction, omitting the full $25,000 overtime deduction and $2,000 charitable deduction. Its rejection of CDCC due to the head's lack of earned income is correct, but its taxable-income base is overstated."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"Its answer is consistent with applying mainly the standard deduction while omitting the full $25,000 overtime deduction and $2,000 charitable deduction. The traced deductions reduce taxable income to $188,944.53 before the $6,600 CTC is applied."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"It incorrectly reduced the liability to zero through unspecified deductions and credits. The deductions leave $188,944.53 taxable income, generating $30,991.80 of pre-credit tax, and the only traced nonrefundable credit is $6,600, leaving $24,391.80."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It invented Arkansas income-tax and medical itemized deductions, included the state refund, and omitted the $25,000 overtime deduction and $2,000 charitable deduction used with the standard deduction. Although it correctly denied CDCC and used a $6,600 CTC, its taxable income was therefore too high."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"It counted only two child tax credits and subtracted a dependent-care credit. All three children qualify for the $2,200 CTC, totaling $6,600, while CDCC is zero because the head has no earned income."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"It correctly used the overtime, standard, charitable, and $6,600 CTC components but incorrectly subtracted a $600 CDCC. The head has no listed earned income, so the joint earned-income limitation makes CDCC zero."
-us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It used itemized charitable and SALT deductions instead of the traced $32,200 standard deduction plus the separate $2,000 charitable deduction. SALT paid is unlisted and therefore zero, while the correct total deduction is $59,200 including the $25,000 overtime deduction."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"Its generic high-income estimate omits the traced $59,200 deduction total and the $6,600 CTC. Applying those items produces $188,944.53 of taxable income and $24,391.80 after nonrefundable credits."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"It used pre-TCJA personal exemptions and 25% brackets, invented Arkansas tax and medical deductions, and omitted the $25,000 overtime deduction. It also wrongly phased out the CTC; the full $6,600 applies at this AGI."
-us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It improperly subtracted the employee's insurance premium from wages, used reverted pre-TCJA brackets and personal exemptions, and invented a state-tax deduction. It omitted the $25,000 overtime deduction and $6,600 CTC while subtracting an unavailable $1,200 CDCC."
-us,scenario_117,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It correctly used the $25,000 overtime deduction, $6,600 CTC, and zero CDCC, but then invented medical, SALT, and miscellaneous itemized deductions. The trace instead subtracts the $32,200 standard deduction and a $2,000 charitable deduction, producing taxable income of $188,944.53."
-us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"It supplied no numeric output or explanation for the requested variable, so the response could not be parsed or evaluated as a substantive calculation."
-us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"It included the state refund and substituted invented Arkansas income-tax and medical itemized deductions for the $32,200 standard deduction. The trace uses AGI of $248,144.53 and deducts $32,200 standard, $25,000 overtime, and $2,000 charity; its zero-CDCC and $6,600-CTC treatments are correct."
-us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"It incorrectly asserted that credits eliminate all federal tax. The traced bracket tax is $30,991.80 and the $6,600 nonrefundable CTC leaves $24,391.80 rather than zero."
-us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"It invented mortgage interest from the mortgage balance and an assumed rate, added an unlisted $10,000 SALT deduction, and included the state refund. It also omitted the $25,000 overtime deduction and used a $6,000 CTC plus an unavailable $600 CDCC instead of the traced $6,600 CTC alone."
-us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It excluded the entire $35,525 overtime premium from wages even though wages already include it, then treated the overtime benefit as additional deductions or credits and invented mortgage-interest and SALT adjustments. The correct treatment starts with $248,675 of wages and allows one capped $25,000 overtime deduction, with no inferred mortgage interest or SALT."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model incorrectly phased down the $25,000 qualified-overtime deduction at this income instead of allowing it in full, and it omitted the separate $2,000 charitable deduction available alongside the standard deduction. It also subtracted a $1,200 CDCC even though the head has no earned income."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the $35,525 overtime premium on top of wages that already include it, included the nontaxable state refund, and treated charity and employee expenses as AGI deductions. It then omitted the full $25,000 overtime deduction, used the wrong standard deduction, and did not correctly subtract the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest, SALT, and a mortgage rate from the loan balance, then itemized those inferred amounts despite the instruction that unlisted numeric inputs are zero. It also used an obsolete $4,500 child-credit amount and subtracted an unavailable CDCC rather than the $6,600 CTC alone."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its stated calculation ends near $25,972, but it submitted $47,714, so the final value does not follow from its own arithmetic. The underlying work also invents mortgage interest and SALT, omits the $25,000 overtime deduction and $2,000 charitable deduction, and uses only $6,000 of CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model fabricated roughly $80,000–$90,000 of itemized deductions from an unspecified mortgage rate and SALT, then stacked the overtime deduction on top. The valid deductions total $59,200, and the only nonrefundable credit is the $6,600 CTC, not an additional $1,200 CDCC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model included the state refund in AGI, omitted the full $25,000 overtime deduction and separate $2,000 charitable deduction, and used a $30,000 rather than $32,200 standard deduction. It also subtracted a $1,200 CDCC despite the head having no earned income and used $6,000 rather than $6,600 of CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"Its detailed computation arrives at about $22,630 but it submitted $62,700, a direct failure to carry its own calculation into the answer. Its calculation also invents mortgage interest and SALT, omits the qualified-overtime deduction, and understates the CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model applied obsolete 15%, 25%, and 28% brackets instead of the applicable 2026 rate schedule. It also omitted the $25,000 overtime deduction, used invented itemized deductions, and wrongly treated the $6,600 CTC as fully phased out."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a pre-TCJA regime with personal exemptions and obsolete rates, while the applicable computation uses the $32,200 standard deduction plus the overtime and charitable deductions. It also understated the CTC and subtracted a CDCC that fails the earned-income limitation."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted amount implies that the model did not reduce AGI to the required $188,944.53 of taxable income using the $32,200 standard deduction, $25,000 overtime deduction, and $2,000 charitable deduction. It also failed to arrive at tax after the full $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model explicitly used pre-TCJA 2026 rules rather than the applicable 2026 brackets and deduction regime. That choice replaces the required standard, overtime, and charitable deductions with the wrong tax structure and therefore cannot produce $30,991.80 before the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,credit_phaseout,False,"The model applied a post-sunset regime with personal exemptions and incorrectly phased out a $1,000-per-child CTC at this income. The household instead receives $2,200 for each of three children, with no phaseout, after tax is computed on $188,944.53."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The answer does not reflect the traced taxable income of $188,944.53 obtained from AGI less $59,200 of deductions. Its generic reference to a CTC phaseout also conflicts with the full $6,600 credit available below the joint phaseout threshold."
+us,scenario_117,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The submitted $41,320 is consistent with taxing a substantially overstated base and failing to apply the traced deductions and credits. The required computation taxes $188,944.53 and then subtracts the full $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used an approximate $31,000 standard deduction but omitted the $25,000 qualified-overtime deduction and $2,000 charitable deduction, while also including the state refund in AGI. Its decision to deny CDCC was correct, but those income and deduction errors overstated taxable income by more than $30,000."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies an overstated taxable-income base rather than the traced $188,944.53 produced by the standard, overtime, and charitable deductions. It also fails to reconcile to $30,991.80 of pre-credit tax less the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The model asserted that deductions and credits eliminate the tax, but the deductions leave $188,944.53 taxable and generate $30,991.80 before credits. The $6,600 CTC reduces that amount only to $24,391.80, not zero."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model itemized estimated Arkansas tax and medical expenses instead of using the traced $32,200 standard deduction plus the $25,000 overtime and $2,000 charitable deductions. It also included the state refund in AGI, leaving taxable income about $26,052 too high."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model allowed only two child tax credits even though all three children are under age 17 and generate $2,200 each. It also subtracted a dependent-care credit despite the head's lack of earned income, so its nonrefundable-credit treatment does not match the sole $6,600 CTC reduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model correctly identified the standard, overtime, charitable, and $6,600 CTC components but then subtracted a $600 CDCC. The head has no earned income, so the earned-income limitation makes CDCC zero; only the $6,600 CTC reduces the $30,991.80 tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used itemized charitable and SALT deductions instead of the $32,200 standard deduction plus the separate $2,000 charitable deduction. The traced deduction total is $59,200 and produces taxable income of $188,944.53 before applying the $6,600 CTC."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The $52,000 answer reflects a gross-income shortcut that did not apply the $59,200 of traced deductions or the $6,600 CTC. The correct tax base is $188,944.53, not an amount near gross wages."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model used a post-sunset regime with personal exemptions and a 25% bracket, and it invented itemized state-tax and medical deductions. It also wrongly phased out the CTC; the applicable computation uses current 2026 brackets and allows the full $6,600 credit."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied obsolete 10%–15%–25% brackets, personal exemptions, and uncapped SALT while omitting the qualified-overtime deduction. It also incorrectly phased out the CTC at $110,000 rather than allowing the full $6,600 under the applicable joint threshold."
+us,scenario_117,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model incorrectly deducted the employee health premium from wages and then used a reverted rate schedule with personal exemptions and inferred state tax. It omitted the $25,000 overtime deduction and $6,600 CTC while subtracting an unavailable $1,200 CDCC."
+us,scenario_117,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model correctly used the overtime deduction and denied CDCC, but it itemized invented medical, SALT, and miscellaneous deductions instead of taking the $32,200 standard deduction plus the $2,000 charitable deduction. That produced taxable income $1,083.53 below the traced $188,944.53 and understated the tax."
+us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive tax calculation."
+us,scenario_117,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model included the state refund in AGI and itemized inferred Arkansas tax and medical deductions, producing taxable income of $191,319.21 instead of $188,944.53. The correct path uses AGI of $248,144.53 less the $32,200 standard deduction, $25,000 overtime deduction, and $2,000 charitable deduction."
+us,scenario_117,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model asserted that credits erase all liability, but tax before credits is $30,991.80 and the available nonrefundable CTC is only $6,600. The remaining liability is therefore $24,391.80 rather than zero."
+us,scenario_117,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model included the state refund and itemized estimated Arkansas income tax and charity instead of using the $32,200 standard deduction plus the separate $2,000 charitable deduction. Those choices produced taxable income around $191,544 rather than the traced $188,944.53."
+us,scenario_117,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model ultimately invented mortgage interest from an unspecified interest rate and itemized it with SALT and charity, while omitting the $25,000 qualified-overtime deduction. It also used only $6,000 of CTC and subtracted a $600 CDCC even though the full CTC is $6,600 and CDCC is zero."
+us,scenario_117,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model treated the overtime premium as excluded wages and then layered on fictional overtime deductions and credits, SALT adjustments, and mortgage-interest limitations. Gross wages remain $248,675; the overtime provision is a single capped $25,000 deduction, and no mortgage interest is deductible without a listed interest amount."
 us,scenario_117,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model incorrectly treated the Additional Child Tax Credit formula as producing a refund even though the household has enough federal income tax liability to use its CTC nonrefundably. It also double-counted the $35,525 FLSA overtime premium despite the prompt stating that annual gross wages already include overtime, then falsely described income of $284,200 as exceeding the $400,000 joint-filer CTC phaseout threshold and invented a $3,750 refundable limit."
 us,scenario_117,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model submitted no value or explanation for federal_refundable_credits, violating the required output contract."
 us,scenario_117,head_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model invented a household-level “young-child context” pathway and transferred the two-year-old child's WIC category to the 44-year-old head. The head has no listed pregnancy, postpartum, or breastfeeding status and is neither an infant nor a child under five; independently, household income exceeds the 185% poverty limit."
@@ -6748,194 +7077,208 @@ us,scenario_117,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The m
 us,scenario_117,payroll_tax,qwen3.8-max,llm_error,payroll_tax_base,False,"The model correctly calculated approximately $11,439 of Social Security tax and $3,606 of Medicare tax with no Additional Medicare Tax, but submitted $20,603 instead of their $15,044.79 sum. The final value adds $5,558.21 unsupported by its own reasoning."
 us,scenario_117,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_117,spouse_wic_eligible,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly transferred a young child's WIC categorical status to the spouse. Living with a WIC-age child does not make an adult eligible; the spouse has no stated pregnancy, postpartum, or breastfeeding status and the household also fails WIC's income test."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model substituted an Arkansas standard deduction for the $24,659.28 itemized deduction used by the return. It also omitted the $60 qualified-individual credits, so its reconstructed taxable income and credit total did not produce $7,984.05."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium on top of gross wages and invented deductions for health premiums, employee expenses, and self-employment tax. It consequently used the wrong AGI and taxable income instead of $248,144.53 and $223,485.25."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented approximately $22,500 of mortgage interest from the mortgage balance and combined it with the full charitable amount to estimate $48,400 of deductions. The applicable itemized deduction is $24,659.28, and its final $11,264 answer also contradicts its own stated estimate near $7,645."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $2,589 state-tax refund in Arkansas income and invented mortgage interest to estimate $48,000 of deductions. It then failed to apply the Arkansas bracket schedule coherently to its own estimated $202,000 taxable income."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to roughly $190,000 by estimating charitable, mortgage-interest, and medical deductions. The traced itemized deduction is $24,659.28, leaving taxable income of $223,485.25, and the model also omitted the $60 qualified-individual credits."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted the separately reported overtime premium from wages, even though gross wages already contain all overtime pay, and it constructed itemized deductions from an inapplicable SALT amount and employee-expense calculation. It also invented a $240 Arkansas child-care credit instead of applying the traced $145 personal credits and $60 qualified-individual credits."
-us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model's own calculation produced approximately $8,086, but it submitted $12,500 without a computation supporting that jump. It also invented mortgage interest from the outstanding balance rather than using the traced $24,659.28 itemized deduction."
-us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used a $5,000 standard deduction rather than the $24,659.28 itemized deduction, overstating taxable income by $19,659.28. It also stopped before subtracting the $145 personal credits and $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used an Arkansas standard deduction and an unexplained bracket-smoothing estimate instead of the $24,659.28 itemized deduction, so it did not calculate the $223,485.25 taxable-income base. It also omitted the $60 qualified-individual credits from the $205 total nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model double-counted the separately listed FLSA overtime premium on top of gross wages and invented deductions for health premiums, employee expenses, and self-employment tax, producing an erroneous AGI and taxable income. It then used incorrect Arkansas rates and $1,050 of unsupported credits instead of $205."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model invented mortgage interest from the mortgage balance and combined it with charitable giving to estimate $48,400 of itemized deductions rather than using $24,659.28. Its final $11,264 also contradicts its own estimated $7,645 calculation and omits the $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the state tax refund in Arkansas income and invented mortgage interest from the outstanding balance, yielding the wrong AGI and deduction base. It then applied the 3.9% schedule inconsistently, since tax on its own stated $202,000 taxable income cannot produce $11,820."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model reduced taxable income to approximately $190,000 through unsupported medical and mortgage-interest deductions rather than using the traced $24,659.28 itemized deduction and $223,485.25 taxable income. It also subtracted only $145, omitting the additional $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model wrongly treated the separately reported overtime premium as an Arkansas income exclusion from wages already included in the $248,675 annual wage total, cutting AGI by $35,525. It also constructed unsupported SALT and employee-expense deductions and a $240 child-care credit instead of applying the traced $24,659.28 itemized deduction and $205 total credits."
+us,scenario_117,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model invented mortgage interest from the mortgage balance and therefore understated taxable income, but then abandoned its own approximately $8,086 bracket calculation and submitted $12,500. The submitted value does not follow from the stated 3.9% rate schedule or its own tax base."
+us,scenario_117,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model applied a $5,000 standard deduction instead of the $24,659.28 itemized deduction, leaving taxable income far too high. It also failed to subtract the $205 of nonrefundable Arkansas credits."
 us,scenario_117,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model used AGI of $250,734 and deductions of $26,641 instead of AGI of $248,144.53 and itemized deductions of $24,659.28. It also subtracted only the $145 personal credits and omitted the $60 qualified-individual credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The unexplained $10,560 does not result from applying the Arkansas married-joint brackets to traced taxable income of $223,485.25. That schedule yields $8,189.05 before the $205 in nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The unexplained $8,768 exceeds the $8,189.05 liability produced by the Arkansas married-joint brackets on $223,485.25 of taxable income. It therefore fails to reproduce the bracket computation and subsequent $205 credit subtraction."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model based its estimate on AGI of approximately $250,734 rather than $248,144.53 and did not identify the $24,659.28 itemized deduction. Applying the traced deduction and credits yields $7,984.05, not $8,623."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted $8,234.12 does not reflect the complete $205 nonrefundable-credit subtraction from the $8,189.05 bracket liability. The applicable credits are $145 for the five personal/dependent credits plus $60 for qualified individuals."
-us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The unexplained $9,260 is inconsistent with the Arkansas married-joint bracket result of $8,189.05 on taxable income of $223,485.25. The model also failed to account for the ensuing $205 of nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an obsolete and internally inconsistent bracket schedule reaching 4.7% and 4.4%. The applicable 2026 schedule produces $8,189.05 on $223,485.25, and the model also omitted the $205 nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $1,005 treats a high-income Arkansas household's liability as modest despite $223,485.25 of taxable income. The Arkansas brackets produce $8,189.05 before $205 of nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated deductions and offsets as eliminating the entire state tax base. They reduce AGI only to $223,485.25 of taxable income, leaving $7,984.05 after nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $25,900 charitable contribution rather than using the traced total itemized deduction of $24,659.28. It also used $160 of credits instead of the correct $205 comprising $145 of personal credits and $60 of qualified-individual credits."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model expressly applied no nonrefundable state credit, omitting both the $145 personal/dependent credits and the $60 qualified-individual credits. Its unspecified deduction and rate estimate also failed to reproduce the $8,189.05 pre-credit liability."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model treated the charitable amount as the operative itemized deduction instead of using the traced $24,659.28 total and did not enumerate the full $205 credit subtraction. Those inputs produce $7,984.05 rather than $8,107."
-us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model came close but did not apply the exact Arkansas bracket and credit computation. Tax on $223,485.25 is $8,189.05, and subtracting the full $205 of personal and qualified-individual credits gives $7,984.05."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The model rounded the liability to $8,000 instead of calculating it. The exact bracket liability is $8,189.05 and the exact nonrefundable-credit subtraction is $205, producing $7,984.05."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a top rate near 4.4% rather than the applicable Arkansas 2026 schedule. On taxable income of $223,485.25, the schedule yields $8,189.05 before $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model started from unsupported AGI of $242,511 and treated the tax as a flat 3.9% calculation after deducting the full charitable contribution. The correct computation uses AGI of $248,144.53, itemized deductions of $24,659.28, graduated brackets, and $205 of credits."
-us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used taxable income near $221,000 after vaguely described standard deductions or exemptions rather than the traced $223,485.25 after itemization. It also failed to apply the exact $205 nonrefundable-credit total."
-us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for the requested output, so the required structured numeric answer was missing."
-us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model chose the joint standard deduction and obtained taxable income near $245,654 instead of itemizing $24,659.28 to reach $223,485.25. It also incorrectly stated that no Arkansas child-care credit applied and omitted the traced $60 qualified-individual credits."
-us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model incorrectly reduced the Arkansas liability to zero. The household retains $223,485.25 of taxable income, generating $8,189.05 before $205 of nonrefundable credits."
-us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $35,525 overtime premium even though the prompt states that gross wages include overtime, inflating income to $286,259. It then used a 4.4% flat rate, chose the standard deduction despite its own itemization discussion, and omitted the $60 qualified-individual credits."
-us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model's stated arithmetic produced $7,127, but it submitted $5,105 without any supporting adjustment. It also used unsupported AGI and deduction figures and credited only $60 rather than the traced $205 total."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The submitted $10,560 does not result from applying the Arkansas schedule to the traced taxable income of $223,485.25. The correct bracket computation is $8,189.05 before the $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The submitted $8,768 implies an incorrect Arkansas bracket computation or credit subtraction. Tax on the traced $223,485.25 base is $8,189.05 before credits and $7,984.05 after the full $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model anchored its calculation to approximately $250,734 of income rather than the traced Arkansas AGI of $248,144.53 and did not identify the exact $24,659.28 itemized deduction. That incorrect base prevented it from reaching taxable income of $223,485.25 and the resulting $7,984.05 liability."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $8,234.12 reflects an incorrect bracket or nonrefundable-credit calculation on Arkansas taxable income. The schedule yields $8,189.05 before credits, followed by $205 of credits, not the submitted amount."
+us,scenario_117,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted $9,260 exceeds the $8,189.05 tax generated by Arkansas's brackets on $223,485.25 of taxable income. It therefore failed to use the correct bracket result and the subsequent $205 nonrefundable-credit subtraction."
+us,scenario_117,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used obsolete and internally inconsistent Arkansas brackets extending through 4.7% and 4.4% instead of the applicable 2026 schedule. It also deducted the full $25,900 donation rather than the traced $24,659.28 itemized amount and omitted all $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The model characterized the liability as modest despite $223,485.25 of Arkansas taxable income and submitted a number consistent with never applying the high-income portion of the state schedule. The bracket calculation alone produces $8,189.05 before the $205 credit subtraction."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated unspecified deductions and offsets as sufficient to eliminate the tax, even though the traced deductions leave $223,485.25 of taxable income. Arkansas tax remains $7,984.05 after the full $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model deducted the full $25,900 charitable contribution instead of using total traced itemized deductions of $24,659.28, producing taxable income of $222,245 rather than $223,485.25. It also used $160 of personal and dependent credits instead of the traced $145 personal credits plus $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model referred to standard and personal deductions rather than the $24,659.28 itemized deduction that produces taxable income of $223,485.25. It also stated that no nonrefundable credit applied, omitting both the $145 personal credits and $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used a loosely estimated charitable deduction instead of the exact $24,659.28 itemized deduction and did not calculate the traced $223,485.25 taxable-income base. It also failed to include the full $205 credit total, including $60 for qualified individuals."
+us,scenario_117,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model came close but did not apply the exact credit composition after the bracket calculation. Arkansas tax is $8,189.05 before credits, and subtracting both $145 of personal credits and $60 of qualified-individual credits yields $7,984.05, not $7,995.31."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The rounded $8,000 estimate skipped the exact Arkansas bracket and credit calculations. The required steps produce $8,189.05 before credits and $7,984.05 after subtracting $205."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model applied a top rate near 4.4% rather than the applicable Arkansas schedule that yields $8,189.05 on $223,485.25 of taxable income. It also did not subtract the traced $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model started from $250,734 rather than $248,144.53 and constructed $26,641 of deductions from medical and charitable amounts instead of using $24,659.28. It then omitted the $145 personal credits and $60 qualified-individual credits by asserting that no nonrefundable credits applied."
+us,scenario_117,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unexplained AGI of $242,511 and treated the full $25,900 donation as the deduction, producing an erroneous $216,611 tax base. It also applied 3.9% as a flat rate and omitted the $205 of nonrefundable credits."
+us,scenario_117,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income near $221,000 rather than deriving the traced $223,485.25 from AGI and itemized deductions. It also omitted the additional $60 qualified-individual credits when allowing only vague personal credits."
+us,scenario_117,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so the answer could not be parsed or evaluated substantively."
+us,scenario_117,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used the joint standard deduction and taxable income near $245,654 instead of the $24,659.28 itemized deduction and $223,485.25 taxable income. It also subtracted only $145 and omitted the $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer treats deductions or credits as eliminating liability, but the traced deductions leave $223,485.25 taxable. The Arkansas brackets produce $8,189.05 before credits, and the available $205 of credits reduce it only to $7,984.05."
+us,scenario_117,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used AGI of $250,734 and deducted the full $25,900 donation, rather than using AGI of $248,144.53 and total itemized deductions of $24,659.28. It also used approximately $155 of credits instead of the exact $205 comprising $145 personal credits and $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model double-counted the $35,525 overtime premium even though it is already part of the stated $248,675 annual gross wages. It then chose a standard-deduction calculation despite recognizing itemization, applied an incorrect flat 4.4% rate, and omitted the $60 qualified-individual credits."
+us,scenario_117,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,other,False,"The model used an unsupported $213,150 income figure and an invented $28,867 filing deduction, then described a $7,127 result while submitting $5,105. It also treated only $60 of dependent credits as applicable instead of subtracting the traced $205 total credits from $8,189.05."
 us,scenario_117,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_118,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model stopped at a generic disability-based Medicaid inquiry and failed to apply New York's automatic SSI-recipient pathway. The head meets SSI's aged criterion and passes the $70 resource and $213.33 countable-income tests, producing a $780.67 monthly SSI benefit and automatic Medicaid eligibility."
 us,scenario_118,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly stated that the $2,800 Social Security retirement benefit is not subject to employee payroll tax and that no wages were provided, but then contradicted that reasoning by assigning $172.90 of employee-side tax. It improperly placed retirement benefits or another nonexistent amount in the payroll-tax base instead of applying the tax rates to a zero earned-income base."
-us,scenario_118,snap,claude-fable-5,llm_error,categorical_eligibility,False,The model omitted the modeled $780.67 monthly SSI payment and calculated SNAP from Social Security alone. That omission drove adjusted income toward zero and incorrectly produced a maximum allotment instead of applying the $58.20 contribution based on $194.38 of net income.
-us,scenario_118,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model reversed the eligibility result despite income, assets, and deductions satisfying every SNAP test, and it failed to apply categorical eligibility through SSI and TANF non-cash eligibility. The resulting allotments sum to $2,903.94, not zero."
-us,scenario_118,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model omitted the modeled SSI payment and therefore applied the shelter deduction against only $233.33 of monthly Social Security, forcing net income to zero. Including total monthly gross income of $1,014 leaves $194.38 of net income and a $58.20 expected contribution."
-us,scenario_118,snap,claude-opus-4.8,llm_error,other,False,"The model's stated arithmetic supports an annual amount near $3,400, but it submitted $2,533 without a computation producing that figure. It also omitted modeled SSI and failed to use the traced $194.38 net income, $58.20 contribution, and month-specific maximum allotments."
-us,scenario_118,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model subtracted about $9 from a $298 maximum but then treated the result as roughly $203, an arithmetic error of about $86 per month. It also omitted modeled SSI, which makes the actual net income $194.38 and the expected contribution $58.20."
-us,scenario_118,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model explicitly asserted that no SSI was present and calculated deductions from Social Security alone. PolicyEngine models $780.67 in monthly SSI, so net income is $194.38 rather than zero and the household does not receive the maximum allotment."
-us,scenario_118,snap,claude-sonnet-5,llm_error,other,False,"The model said net income was at or below zero and the maximum benefit was $3,504, then reduced the answer to $2,555 without identifying any countable income or contribution supporting that reduction. The traced calculation instead uses $194.38 of net income and a $58.20 monthly contribution."
-us,scenario_118,snap,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model omitted the modeled SSI payment, set net income to zero, and used the FY2025 maximum allotment for every month. Total gross income is $1,014 per month, and the $194.38 net amount produces a positive expected contribution rather than a maximum benefit."
-us,scenario_118,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included SSI but incorrectly reduced $1,098 of stated gross income to zero through shelter and utility deductions. The traced inputs produce $1,014 of gross income and $194.38 of net income, so $58.20 must be deducted from the monthly maximum allotment."
-us,scenario_118,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated low income and sparse expense information as preventing a benefit calculation. The household satisfies the net-income, gross-income, asset, and categorical-eligibility tests, and the resulting monthly allotments sum to $2,903.94."
-us,scenario_118,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an incorrect SSI amount and stopped the deduction calculation at $679 of net income. PolicyEngine uses $1,014 of monthly gross income and deductions that reduce countable net income to $194.38, yielding a $58.20 contribution rather than $204."
-us,scenario_118,snap,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated NYSNIP participation as guaranteeing the maximum allotment and used a flat $291 monthly parameter. Categorical eligibility establishes eligibility but does not eliminate the 30% reduction for $194.38 of net income, and the applicable maximum and deduction parameters also change during the year."
-us,scenario_118,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The submitted $3,492 is a flat $291 maximum allotment for all twelve months and contains no 30% net-income reduction. The correct computation subtracts a $58.20 expected contribution and accounts for the within-year parameter update."
-us,scenario_118,snap,gemini-3.7-flash,llm_error,other,False,"The model gave no calculation connecting the acknowledged deductions to its $69 monthly implied benefit. The traced calculation produces monthly allotments of $239.80 for most months and $248.58 for others, totaling $2,903.94."
-us,scenario_118,snap,glm-5.2,llm_error,categorical_eligibility,False,The model calculated income from Social Security alone and omitted $780.67 of modeled monthly SSI. This incorrectly reduced net income to zero and converted an estimated maximum allotment directly into the annual answer.
-us,scenario_118,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model incorrectly asserted that the supplied household facts do not generate SNAP despite extremely low assets and income. SSI and TANF non-cash categorical eligibility applies, and all income and asset tests are satisfied."
-us,scenario_118,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model treated the absence of additional supporting facts as a bar to SNAP instead of applying the stated zero defaults and assumed take-up. PolicyEngine derives SSI categorical eligibility and a positive allotment from the supplied facts.
-us,scenario_118,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model omitted modeled SSI, invented a $20 SSI/SNAP exclusion that reduced net income to zero, and used an obsolete $243 monthly maximum. The calculation instead has $194.38 of net income and maximum allotments of $298 or an updated amount depending on the month."
-us,scenario_118,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model described only Social Security and the standard deduction, omitting modeled SSI and the full traced deduction calculation. Its implied $290.70 monthly benefit does not subtract the $58.20 contribution from the applicable monthly maximums."
-us,scenario_118,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model's estimated $115 monthly benefit implies substantially overstated net income after deductions. The applicable deductions reduce $1,014 of gross monthly income to $194.38, producing allotments near $240 rather than $115."
-us,scenario_118,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model counted Social Security and SSI but derived an unsupported $123 monthly allotment, which reflects excessive countable net income. The traced deductions leave $194.38 of net income and a $58.20 contribution, producing monthly benefits of $239.80 or $248.58."
-us,scenario_118,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model refused to calculate SNAP despite the prompt supplying the household facts and directing unlisted inputs to zero. Those facts generate modeled SSI, categorical eligibility, and a positive annual allotment of $2,903.94."
-us,scenario_118,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model correctly recognized SSI categorical eligibility but left approximately $735 of net income after claiming unlimited shelter deductions. The traced deduction calculation leaves only $194.38 of net income, so the contribution is $58.20 rather than roughly $220."
-us,scenario_118,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,The model omitted modeled SSI and incorrectly declared shelter costs unlisted even though mortgage interest and real-estate taxes were supplied. It consequently used only $40 of net income and obsolete estimated parameters instead of the traced $194.38 net income and month-specific allotments.
-us,scenario_118,snap,inkling,llm_error,categorical_eligibility,False,"The model omitted modeled SSI and therefore reduced net income to zero using shelter expenses. With SSI included, net income is $194.38 and the household owes a $58.20 expected contribution rather than receiving $298 in every month."
-us,scenario_118,snap,kimi-k2.6,llm_error,categorical_eligibility,False,"The model calculated deductions from Social Security alone, omitted modeled SSI, and selected an unsupported $309 maximum allotment. Including SSI leaves positive net income and requires the 30% benefit reduction."
-us,scenario_118,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model counted SSI but refused to treat the listed mortgage interest and property taxes as SNAP shelter expenses, leaving net income at $892. The traced deductions reduce net income to $194.38, so the expected contribution is $58.20 rather than $268."
-us,scenario_118,snap,minimax-m3,llm_error,thresholds_rates,False,"The model inverted the SNAP benefit formula by treating very low income after deductions as producing zero rather than a benefit near the maximum. SNAP subtracts 30% of net income from the maximum allotment, so lower net income increases the benefit."
-us,scenario_118,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI and ignored the excess-shelter deduction while calculating a near-maximum benefit from Social Security alone. The full calculation uses $1,014 of gross monthly income, $194.38 of net income, and a $58.20 contribution."
-us,scenario_118,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly imposed a qualifying-child requirement on SNAP. A single elderly or disabled adult can qualify, and this household satisfies the income, asset, and categorical-eligibility rules."
-us,scenario_118,ssi,claude-fable-5,llm_error,thresholds_rates,False,"The model explicitly derived the correct $994 − $213.33 = $780.67 monthly benefit and $9,368 annual result, then discarded it and submitted an unexplained $6,144. It failed to carry its own final SSI computation into the output."
-us,scenario_118,ssi,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model compared $2,800 of annual Social Security with a monthly SSI maximum and falsely declared the person income-ineligible. Social Security is $233.33 monthly, and after the $20 monthly exclusion only $213.33 reduces the $994 maximum, leaving a positive benefit."
-us,scenario_118,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $967 federal rate instead of the 2026 $994 monthly maximum and added an unsupported New York supplement. The requested SSI amount is the $994 maximum reduced by $213.33 of monthly countable Social Security, with no added state supplement."
-us,scenario_118,ssi,claude-opus-4.8,llm_error,missing_output,False,"The model correctly concluded that categorical and resource eligibility were satisfied and that the SSI payment was positive, but submitted zero. It also treated the $20 general exclusion as annual rather than monthly; the annual exclusion is $240, leaving $2,560 countable income and $9,368 of SSI."
-us,scenario_118,ssi,claude-opus-5,llm_error,thresholds_rates,False,"The model correctly found $213.33 of monthly countable income but replaced the resulting benefit with an unexplained estimate of roughly $667 per month. Applying that income to the $994 maximum gives $780.67 per month, not $667, and no New York supplement adjustment changes this output."
-us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used an estimated $967 monthly federal rate rather than the 2026 $994 maximum. It also inconsistently converted the $20 monthly exclusion into only $20 annually before arriving at $8,824; the correct annual countable income is $2,560 and the benefit is $9,368."
-us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"The model's stated arithmetic produced $12,152 − $2,780 = $9,372, but it then submitted $6,212 through an unsupported adjustment. It also used the wrong maximum and only a $20 annual exclusion; the trace uses a $994 monthly maximum and a $240 annual exclusion to produce $9,368."
-us,scenario_118,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,The model used a $991 federal maximum instead of $994 and added a $54 monthly New York supplement that is not part of the requested SSI output. The correct monthly computation is $994 − $213.33 = $780.67.
-us,scenario_118,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model added an $87 monthly New York supplement and used a $991 federal rate. PolicyEngine's SSI output uses the $994 monthly maximum without that supplement, reduced by $213.33 of countable income."
-us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model falsely stated that $2,800 of annual Social Security exceeds the individual SSI maximum. Converted to $233.33 monthly and reduced by the $20 monthly exclusion, it leaves only $213.33 of monthly countable income against the $994 maximum."
-us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of annual countable income but used a $943 monthly federal maximum and added a $1,044 New York supplement. The applicable maximum is $994 monthly, with no state supplement added, yielding $11,928 − $2,560 = $9,368."
-us,scenario_118,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model substituted the 2024 $943 monthly rate for the 2026 $994 rate and then added an $87 monthly New York supplement. The requested output instead subtracts $2,560 of annual countable income directly from the $11,928 annual 2026 maximum."
-us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly obtained $2,560 in annual countable income but implicitly used an $11,796 annual maximum. The 2026 maximum is $994 monthly, or $11,928 annually, so the benefit is $11,928 − $2,560 = $9,368."
-us,scenario_118,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly found $2,560 of annual countable income but used an inflated combined federal-and-New-York maximum. The requested SSI output uses the $11,928 annual maximum without a New York supplement, producing $9,368."
-us,scenario_118,ssi,glm-5.2,llm_error,thresholds_rates,False,"The model used an estimated $991 federal rate instead of $994 and added a $94 monthly New York supplement. With no state supplement in this output, $994 minus $213.33 of countable monthly income equals $780.67."
-us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model ignored the stated age, blindness, disability, $70 resource balance, and Social Security amount and declared that positive SSI was not established. Those facts establish categorical and resource eligibility, while the income calculation leaves a $780.67 monthly payment."
-us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed that qualifying age/disability and detailed resource facts were absent even though age 74, blindness, disability, and $70 in bank assets were expressly listed. After the $20 monthly income exclusion, the listed Social Security reduces rather than eliminates the $994 monthly benefit."
-us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly applied the $240 annual exclusion but used a $982.67 monthly maximum instead of $994. The correct annual maximum is $11,928, and subtracting $2,560 yields $9,368."
-us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"The model deducted only $20 from the entire year's Social Security income, producing $2,780 of countable income. The general exclusion is $20 each month, or $240 annually, so countable income is $2,560 and $11,928 − $2,560 = $9,368."
-us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"The model wrongly asserted that SSI calculation inputs were absent. The prompt supplied qualifying age, blindness, disability, $70 of resources, and $2,800 of annual Social Security, which produce $9,368 after the monthly exclusion and income reduction."
-us,scenario_118,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used a $991 federal rate and added an $87 monthly New York living-alone supplement. The applicable SSI maximum is $994 per month without that supplement, reduced by $213.33 of countable monthly income."
-us,scenario_118,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the 2024 $943 federal maximum as a proxy for 2026 and added an $87 monthly New York supplement. The benchmark requires the 2026 $994 maximum without the supplement, yielding $780.67 monthly after the income reduction."
-us,scenario_118,ssi,inkling,llm_error,thresholds_rates,False,"The model correctly calculated roughly $780.67 of monthly SSI from the $994 maximum and countable Social Security, then improperly added an $87 monthly New York supplement. The requested PolicyEngine SSI output is the federal computation alone, totaling $9,368 annually."
-us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of annual countable income but used a $991 monthly maximum. The 2026 maximum is $994 per month, or $11,928 annually, leaving $9,368."
-us,scenario_118,ssi,kimi-k3,llm_error,thresholds_rates,False,"The model explicitly derived the correct federal SSI amount of $9,368, then added a $1,044 New York living-alone supplement. That supplement is not included in the requested SSI output."
-us,scenario_118,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model reversed its own comparison by claiming that $233 of monthly Social Security exceeds a benefit rate it identified as $967 monthly. After the $20 exclusion, $213.33 is countable against the actual $994 maximum, so SSI remains positive."
-us,scenario_118,ssi,qwen-3.7-max,llm_error,missing_output,False,"The model calculated a positive monthly SSI benefit and an $8,756 annual estimate but then submitted zero. It also used the obsolete $943 maximum; applying the 2026 $994 rate to $213.33 of monthly countable income yields $9,368 annually."
-us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model declared no positive SSI despite the head satisfying the aged, blind, and disabled category and holding only $70 in countable resources. The $2,800 annual Social Security amount leaves $2,560 countable after the monthly general exclusion, so the $11,928 annual maximum produces $9,368."
-us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,categorical_eligibility,False,"The model incorrectly excluded homeowners from the real property tax credit pathway. The taxpayer's age of 74 and $1,633.50 of real estate taxes qualify the household for the refundable $375 credit despite having no earned or taxable income."
-us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,categorical_eligibility,False,The model overlooked New York's refundable real property tax credit by considering only credits tied to earnings or families. This elderly homeowner's age and property taxes generate a $375 refundable credit.
-us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,thresholds_rates,False,The model identified the real property tax credit and its $375 senior maximum but replaced the applicable capped result with an unsupported $63 approximation. The qualifying property-tax calculation reaches the $375 cap.
-us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,categorical_eligibility,False,The model wrongly treated positive taxable income or tax liability as necessary for the refundable real property tax credit. Refundability allows the qualifying 74-year-old homeowner to receive $375 with zero New York taxable income.
-us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,categorical_eligibility,False,"The model treated the absence of rent and taxable income as disqualifying, omitting the homeowner pathway based on real estate taxes paid. The taxpayer qualifies through age 74 and $1,633.50 of property taxes, yielding $375."
-us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied the $75 maximum for a younger single taxpayer instead of the $375 maximum applicable to a qualifying taxpayer age 65 or older. Because the computed property-tax relief exceeds that senior cap, the credit is $375."
-us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly dismissed the homeowner real property tax credit because no rent was paid and characterized the senior benefit as negligible. The listed property taxes activate the homeowner pathway, and the age-based maximum yields $375."
-us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model correctly included the $375 real property tax credit but incorrectly added a $75 New York household credit to refundable credits. The household credit offsets tax as a nonrefundable credit and does not belong in state_refundable_credits.
-us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model denied eligibility without applying New York's real property tax credit rules. Age 74 and $1,633.50 of homeowner real estate taxes satisfy the relevant pathway and produce $375."
-us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,categorical_eligibility,False,"The model limited refundable credits to the EITC and child-related programs and omitted the real property tax credit. That credit requires neither earned income nor dependents, and the senior homeowner receives $375."
-us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,categorical_eligibility,False,The model failed to evaluate the refundable real property tax credit. The taxpayer's senior status and reported real estate taxes generate the $375 credit.
-us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,categorical_eligibility,False,The model incorrectly made earned income or qualifying dependents prerequisites for every refundable New York credit. The real property tax credit has a separate elderly-homeowner pathway that yields $375 here.
-us,scenario_118,state_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used an inapplicable 3.5% subtraction formula and an unsupported expanded $1,000 maximum. The applicable real property tax credit computation is capped at $375 for this qualifying senior household."
-us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model overlooked the listed age and real estate taxes that trigger New York's refundable real property tax credit. Those facts produce a $375 credit.
-us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,categorical_eligibility,False,The model failed to recognize the real property tax credit indicated by the taxpayer's age and homeowner property-tax payment. The qualifying refundable amount is $375.
-us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,categorical_eligibility,False,The model incorrectly inferred that Social Security-only income prevents all refundable New York credits. The real property tax credit is available independently of earned income and yields $375 for this senior homeowner.
-us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model substituted a $90 New York household credit for the applicable refundable real property tax credit. The household credit is not part of refundable credits, while the taxpayer's age and property taxes produce a refundable $375 real property tax credit."
-us,scenario_118,state_refundable_credits,grok-4.3,llm_error,categorical_eligibility,False,"The model missed the elderly-homeowner eligibility pathway for New York's real property tax credit. Age 74 and $1,633.50 in real estate taxes generate $375."
-us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,categorical_eligibility,False,The model treated the EITC and child credits as the only relevant refundable programs and omitted the real property tax credit. That separate homeowner credit requires no earnings or children and supplies $375.
-us,scenario_118,state_refundable_credits,inkling,llm_error,categorical_eligibility,False,The model incorrectly concluded that no property tax credit applies despite the explicit age and real estate-tax facts. The senior homeowner qualifies for a $375 refundable real property tax credit.
-us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,categorical_eligibility,False,"The model searched only earnings-, child-, tuition-, and childcare-based credits and ignored New York's real property tax credit. The age-74 homeowner's $1,633.50 property-tax payment produces $375."
-us,scenario_118,state_refundable_credits,minimax-m3,llm_error,categorical_eligibility,False,The model wrongly required New York tax liability or qualifying children for a refundable credit. The real property tax credit is refundable against zero liability and yields $375 based on senior age and property taxes.
-us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,categorical_eligibility,False,"The model limited eligibility factors to taxable earnings and children, omitting the taxpayer's age and homeowner property taxes. Those facts qualify the household for the $375 real property tax credit."
-us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,The model incorrectly rejected all refundable credits despite the taxpayer being 74 and paying real estate taxes. New York's elderly-homeowner real property tax credit applies and equals $375.
-us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated 205% FPL as the controlling Virginia FAMIS cutoff and concluded that the household was below it. Under the engine's applicable CHIP income test, child1 passes the age condition but fails the income limit."
-us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,The model asserted that household income was within Virginia's CHIP threshold without applying the engine's applicable income limit. Child1 is age-qualified but income-ineligible.
-us,scenario_119,child1_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,The model used an upper limit of 205% FPL and placed household MAGI beneath it. The engine's applicable Virginia CHIP income test excludes child1 even though age 14 satisfies the age rule.
-us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model assumed that income above the Medicaid child limit necessarily fell inside Virginia's CHIP band. It failed the separate CHIP upper-income test, which leaves child1 ineligible despite Medicaid category NONE."
-us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,The model relied on an estimated Virginia CHIP income range instead of the applicable engine threshold. Child1 meets the under-19 condition but fails the CHIP income test.
-us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model calculated household MAGI as approximately 192% FPL and therefore placed it below a 200% cutoff. That income-to-FPL calculation is wrong under the engine's household-income computation, which puts child1 outside the CHIP income limit."
-us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated CHIP MAGI as $46,180 and 168% FPL, apparently subtracting amounts that do not reduce the program's household-income measure. The engine uses approximately $55,800 and finds that child1 fails the applicable CHIP income limit."
-us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric value or explanation for child1_chip_eligible, violating the required output contract."
-us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used MAGI of $52,569 and placed it within Virginia's CHIP band. The engine's income measure is approximately $55,800, which fails the applicable CHIP upper-income test; potential eligibility rather than enrollment does not change that result."
+us,scenario_118,snap,claude-fable-5,llm_error,taxable_income_or_deductions,False,The model counted only the listed Social Security and omitted the $780.67 monthly SSI generated for this household from SNAP gross income. That omission drove net income to zero and incorrectly awarded the maximum allotment instead of subtracting the $58.20 expected contribution.
+us,scenario_118,snap,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model inverted the SNAP income test: income of $1,014 per month and net income of $194.38 satisfy the applicable limits, and SSI receipt also supplies categorical eligibility. Its conclusion that extremely low income prevents eligibility has no basis in the SNAP allotment formula."
+us,scenario_118,snap,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model omitted the household’s modeled $780.67 monthly SSI and calculated deductions from Social Security alone. Correct gross income is $1,014 per month, leaving $194.38 of net income and a $58.20 expected contribution rather than a maximum allotment."
+us,scenario_118,snap,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model based its calculation on Social Security alone, omitted modeled SSI from gross income, and then submitted $2,533 despite its own arithmetic pointing to roughly $3,400. The correct calculation uses $1,014 monthly gross income, $194.38 net income, and the resulting 30% contribution."
+us,scenario_118,snap,claude-opus-5,llm_error,thresholds_rates,False,"The model stated that $298 minus 30% of $29 was about $203, an arithmetic error of roughly $86 per month. It also omitted modeled SSI from gross income; the correct expected contribution is $58.20 from $194.38 of net income."
+us,scenario_118,snap,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated Social Security as the household’s only SNAP income and therefore reduced net income to zero with the shelter deduction. Modeled SSI raises gross income to $1,014 per month, and the applicable deductions leave $194.38 rather than zero."
+us,scenario_118,snap,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI and asserted that deductions reduced net income to zero, then applied an unexplained reduction to its stated maximum-benefit result. SNAP net income is $194.38, and the benefit follows directly from the $58.20 expected contribution."
+us,scenario_118,snap,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,The model omitted the $780.67 monthly SSI generated for the household and consequently set net income to zero. It also used the $292 prior-year maximum instead of the monthly 2026 parameters reflected in the annual calculation.
+us,scenario_118,snap,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model recognized SSI income but used the wrong SSI amount and over-deducted shelter expenses until net income became zero. The trace yields $1,014 gross income and $194.38 net income, so the household owes a $58.20 expected contribution rather than receiving the maximum."
+us,scenario_118,snap,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model treated low income and listed expenses as preventing a calculable benefit. The household passes the income and asset tests, receives categorical eligibility through SSI and TANF non-cash eligibility, and has a positive allotment under the standard 30% formula."
+us,scenario_118,snap,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model used an incorrect SSI amount and limited shelter costs to mortgage interest and property tax, omitting the shelter allowance used by the SNAP computation. Those errors left $679 of net income instead of $194.38 and overstated the expected contribution."
+us,scenario_118,snap,gemini-3.5-flash,llm_error,state_local_rule,False,"The model replaced the ordinary traced allotment calculation with an asserted NYSNIP maximum-benefit rule. PolicyEngine applies the 30% reduction to $194.38 of net income, and the annual total also reflects parameter changes across the calendar year."
+us,scenario_118,snap,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model supplied no derivation for its $291 monthly figure and treated the annual benefit as twelve identical months. The traced calculation uses a $298 maximum for most months, subtracts $58.20, and includes some $248.58 months after parameter updates."
+us,scenario_118,snap,gemini-3.7-flash,llm_error,thresholds_rates,False,"The answer implies a $69 monthly allotment despite stating that the household qualifies after its deductions. With $194.38 of net income, the 30% contribution is $58.20 and the normal monthly allotment is $239.80, not $69."
+us,scenario_118,snap,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI from SNAP gross income, allowing the shelter deduction to reduce income to zero. It then guessed a $295 maximum rather than applying the traced $298 maximum and $58.20 expected contribution."
+us,scenario_118,snap,gpt-5.4-mini,llm_error,categorical_eligibility,False,The model incorrectly asserted that the facts do not generate SNAP despite very low assets and income. The household satisfies every eligibility test and is categorically eligible through modeled SSI receipt and TANF non-cash eligibility.
+us,scenario_118,snap,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the absence of additional supporting facts as disqualifying even though unlisted inputs are zero and the supplied facts fully determine eligibility. The household passes the net, gross, and asset tests and receives categorical eligibility."
+us,scenario_118,snap,gpt-5.5,llm_error,thresholds_rates,False,"The model used a nonexistent $20 SSI/SNAP exclusion to set net income to zero and an obsolete $243 maximum allotment. The applicable computation produces $194.38 of net income, a $58.20 contribution, and a $239.80 normal monthly allotment."
+us,scenario_118,snap,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model described only Social Security and the standard deduction, omitting modeled SSI and the complete shelter-deduction computation. Its answer is consequently close to a maximum benefit rather than the reduced allotment based on $194.38 of net income."
+us,scenario_118,snap,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model gave no calculation supporting its $115 monthly estimate and did not identify the amounts used for SSI or shelter deductions. The traced inputs yield $194.38 of net income and about $239.80 for a normal month, not $115."
+us,scenario_118,snap,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model counted Social Security and SSI but understated the shelter deduction by relying only on the listed ownership costs. The full deduction computation leaves $194.38 of net income and a normal allotment of $239.80, not $123."
+us,scenario_118,snap,grok-4.3,llm_error,categorical_eligibility,False,"The model wrongly treated the SNAP amount as indeterminate from the supplied facts. PolicyEngine generates SSI, applies categorical eligibility and the income deductions, and computes a positive benefit through the 30% contribution formula."
+us,scenario_118,snap,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model acknowledged categorical eligibility but left approximately $735 of net income after claiming to apply unlimited shelter deductions. The actual deductions reduce $1,014 of gross income to $194.38, producing a $58.20 contribution rather than roughly $220."
+us,scenario_118,snap,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI when concluding that deductions reduced net income to zero. Correct net income is $194.38, so the household receives the maximum less a $58.20 expected contribution."
+us,scenario_118,snap,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared shelter costs unlisted even though mortgage interest and real estate taxes were explicitly supplied, and it also omitted modeled SSI income. It then used old estimated deduction and allotment parameters instead of the 2026 monthly values."
+us,scenario_118,snap,inkling,llm_error,taxable_income_or_deductions,False,"The model counted only Social Security, so the shelter deduction drove net income to zero and produced twelve maximum allotments. Modeled SSI raises gross income to $1,014 and leaves $194.38 after deductions, requiring a $58.20 contribution."
+us,scenario_118,snap,kimi-k2.6,llm_error,taxable_income_or_deductions,False,The model omitted modeled SSI and therefore reduced net income to zero. It compounded that error by guessing a $309 maximum allotment instead of using the traced monthly maximum and expected contribution.
+us,scenario_118,snap,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly included modeled SSI but wrongly said no SNAP housing-cost input was supplied and allowed only the standard deduction. Mortgage interest, property tax, and the applicable shelter calculation reduce net income to $194.38 rather than $892."
+us,scenario_118,snap,minimax-m3,llm_error,thresholds_rates,False,"The model reversed the benefit formula by claiming that income below the allotment threshold produces zero after deductions. SNAP instead pays the maximum allotment minus 30% of net income, yielding a positive benefit when net income is $194.38."
+us,scenario_118,snap,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model correctly reconstructed $1,014 of gross income but understated the shelter deduction by using only mortgage interest and property tax. The full shelter computation reduces net income to $194.38 rather than $608.75, cutting the expected contribution to $58.20."
+us,scenario_118,snap,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted modeled SSI and ignored the explicit mortgage-interest and property-tax shelter deduction, computing from Social Security less the standard deduction alone. It also used estimated $198 and $291 parameters rather than the traced 2026 monthly values."
+us,scenario_118,snap,qwen3.8-max,llm_error,categorical_eligibility,False,"The model invented a qualifying-child requirement for SNAP. A one-person elderly household is a valid SNAP unit, and this household passes the income and asset tests while also receiving categorical eligibility through SSI and TANF non-cash eligibility."
+us,scenario_118,ssi,claude-fable-5,llm_error,other,False,"The model explicitly derived the correct $780.67 monthly and $9,368 annual benefit, then submitted an unrelated $6,144 figure based on a rate it had already identified as wrong."
+us,scenario_118,ssi,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model compared $2,800 of annual Social Security income with a monthly SSI limit and incorrectly declared the head ineligible. Monthly countable income is only $213.33 after the $20 exclusion, so SSI remains payable."
+us,scenario_118,ssi,claude-opus-4.7,llm_error,thresholds_rates,False,"The model used a $967 federal rate instead of the 2026 $994 rate and added an unsupported New York supplement. This output is the federal SSI amount: $994 minus $213.33 monthly countable income produces $9,368 annually."
+us,scenario_118,ssi,claude-opus-4.8,llm_error,other,False,"The model correctly concluded that the resource test is met and that federal SSI is positive, but then submitted zero. Applying the $20 monthly exclusion and the $994 monthly maximum yields $9,368."
+us,scenario_118,ssi,claude-opus-5,llm_error,other,False,"The model stated the correct categorical eligibility and monthly countable income but replaced the resulting calculation with an unexplained $667 monthly estimate. The stated inputs produce $780.67 per month, not $667."
+us,scenario_118,ssi,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used the obsolete $967 monthly federal rate rather than the 2026 $994 rate. With $213.33 of monthly countable Social Security income, the correct annual federal SSI is $9,368."
+us,scenario_118,ssi,claude-sonnet-5,llm_error,other,False,"The model's own base-minus-income arithmetic produced approximately $9,372, but it then imposed an unexplained adjustment to $6,212. New York supplementation and asset rounding do not transform the traced federal calculation of $11,928 minus $2,560."
+us,scenario_118,ssi,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used a $991 federal maximum and added a $54 New York supplement. The SSI output instead uses the $994 monthly maximum without that supplement, yielding $9,368."
+us,scenario_118,ssi,gemini-3-flash-preview,llm_error,thresholds_rates,False,"The model used a $991 federal rate and added an $87 New York supplement to the SSI output. The applicable maximum is $994 monthly with no state supplement added, less $213.33 of monthly countable income."
+us,scenario_118,ssi,gemini-3.1-flash-lite-preview,llm_error,categorical_eligibility,False,"The model incorrectly said $2,800 of annual Social Security exceeds the SSI maximum. After converting income to monthly terms and applying the $20 monthly exclusion, countable income is $213.33 against a $994 monthly maximum."
+us,scenario_118,ssi,gemini-3.1-pro-preview,llm_error,thresholds_rates,False,"The model substituted a $943 monthly federal maximum and added a $1,044 New York supplement. The traced 2026 SSI maximum is $11,928 annually without that supplement, so subtracting $2,560 gives $9,368."
+us,scenario_118,ssi,gemini-3.5-flash,llm_error,thresholds_rates,False,The model used the 2024 $943 federal rate as a proxy and added an $87 monthly New York supplement. The 2026 calculation uses $994 monthly and no state supplement in this output.
+us,scenario_118,ssi,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model correctly derived $2,560 of annual countable income but used an annual maximum of $11,796. The applicable maximum is $994 per month, or $11,928 annually, producing $9,368."
+us,scenario_118,ssi,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of countable income but inflated the benefit by including a New York supplement. The SSI output uses the $11,928 federal maximum alone, yielding $9,368."
+us,scenario_118,ssi,glm-5.2,llm_error,thresholds_rates,False,"The model used a projected $991 federal rate and added a $94 monthly New York supplement. The engine applies the $994 monthly maximum without that supplement, less $213.33 monthly countable income."
+us,scenario_118,ssi,gpt-5.4-mini,llm_error,categorical_eligibility,False,"The model disregarded the supplied age, blindness, disability, income, and resource facts and treated eligibility as unestablished. Age 74 independently satisfies the aged category, $70 passes the resource test, and the income calculation leaves a positive benefit."
+us,scenario_118,ssi,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model claimed the prompt lacked qualifying age, disability, income, and resource facts even though all were explicitly listed. The head qualifies as aged, blind, and disabled, has only $70 of resources, and receives $9,368 after the income reduction."
+us,scenario_118,ssi,gpt-5.5,llm_error,thresholds_rates,False,"The model correctly calculated $2,560 of annual countable income but used a $982.67 monthly maximum. The 2026 maximum is $994 per month, making the annual benefit $11,928 minus $2,560."
+us,scenario_118,ssi,gpt-5.6-terra,llm_error,period_annualization,False,"The model treated the $20 general exclusion as a single annual exclusion and deducted only $20 from annual Social Security income. It applies each month, totaling $240 annually and leaving $2,560 countable rather than $2,780."
+us,scenario_118,ssi,grok-4.3,llm_error,categorical_eligibility,False,"The model ignored the explicit age, blindness, disability, bank assets, and Social Security inputs and set SSI to zero. Those facts establish categorical and resource eligibility and support the full traced income calculation."
+us,scenario_118,ssi,grok-4.5,llm_error,thresholds_rates,False,"The model used a $991 federal rate and added an $87 New York living-alone supplement. The SSI output uses $994 monthly without that supplement, producing $9,368 after countable income."
+us,scenario_118,ssi,grok-4.6,llm_error,thresholds_rates,False,"The model correctly applied the $240 annualized exclusion but used a federal maximum of $11,604 rather than $11,928. Subtracting $2,560 from the correct maximum yields $9,368."
+us,scenario_118,ssi,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the 2024 $943 maximum as a proxy for 2026 and added an $87 New York supplement. The applicable 2026 maximum is $994 monthly, and this output does not add the supplement."
+us,scenario_118,ssi,inkling,llm_error,thresholds_rates,False,"The model correctly derived approximately $780.67 of monthly federal SSI, then added an $87 New York supplement. The requested SSI output remains $780.67 monthly, or $9,368 annually."
+us,scenario_118,ssi,kimi-k2.6,llm_error,thresholds_rates,False,"The model correctly derived $2,560 of annual countable income but used a $991 monthly maximum. The applicable $994 monthly maximum raises the annual result from $9,332 to $9,368."
+us,scenario_118,ssi,kimi-k3,llm_error,thresholds_rates,False,"The model derived the correct $9,368 federal SSI and then added a $1,044 New York living-alone supplement. That supplement is not part of this SSI output."
+us,scenario_118,ssi,minimax-m3,llm_error,categorical_eligibility,False,"The model reversed its own comparison by asserting that $233 of monthly Social Security exceeds a monthly SSI maximum near $967. After the exclusion, $213.33 is below the $994 maximum, so SSI is payable."
+us,scenario_118,ssi,qwen-3.7-max,llm_error,other,False,"The model calculated a positive $8,756 annual benefit under its chosen rate but submitted zero. Using the actual 2026 $994 monthly maximum increases that positive benefit to $9,368."
+us,scenario_118,ssi,qwen3.8-max,llm_error,categorical_eligibility,False,"The model asserted no positive SSI despite explicit aged, blind, disabled, low-resource, and low-income facts. The head passes categorical and resource eligibility, and countable income reduces rather than eliminates the $994 monthly maximum."
+us,scenario_118,state_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model incorrectly excluded a homeowner from the real property tax credit pathway. The head’s age and $1,633.50 of real estate taxes qualify the household for the refundable $375 senior credit despite zero New York taxable income."
+us,scenario_118,state_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,The model overlooked New York’s refundable real property tax credit and considered only credits tied to earnings or families. The 74-year-old homeowner’s property taxes generate a $375 refundable credit.
+us,scenario_118,state_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,The model identified the real property tax credit but substituted an unsupported approximate formula that yielded $63. The applicable senior computation reaches the $375 maximum.
+us,scenario_118,state_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,The model incorrectly treated positive taxable income or tax liability as necessary for the refundable real property tax credit. The qualifying senior homeowner receives $375 even though Social Security is exempt and state tax liability is zero.
+us,scenario_118,state_refundable_credits,claude-opus-5,llm_error,state_local_rule,False,"The model treated the absence of rent and taxable New York income as disqualifying. Homeowners qualify through real estate taxes paid, and this 74-year-old homeowner receives the $375 real property tax credit."
+us,scenario_118,state_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model applied the $75 maximum associated with a non-senior household instead of the $375 maximum available when a household member is age 65 or older. At age 74, the head satisfies the senior condition, so the calculated credit is capped at $375."
+us,scenario_118,state_refundable_credits,claude-sonnet-5,llm_error,state_local_rule,False,"The model wrongly dismissed the homeowner credit because there was no rent and characterized the senior credit as negligible. The listed real estate taxes establish the homeowner pathway, and the senior calculation yields $375."
+us,scenario_118,state_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model correctly included the $375 real property tax credit but incorrectly added a $75 New York household credit to refundable credits. The household credit is not part of this refundable-credit output, so only $375 is included; it also invented SSI income absent from the facts."
+us,scenario_118,state_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The answer implies the model failed to apply New York’s senior homeowner real property tax credit. Age 74 and $1,633.50 of real estate taxes produce a refundable credit of $375."
+us,scenario_118,state_refundable_credits,gemini-3.5-flash,llm_error,state_local_rule,False,"The model limited its review to earned-income and child credits. It omitted the refundable real property tax credit, which gives this qualifying senior homeowner $375."
+us,scenario_118,state_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,The model failed to recognize the refundable New York real property tax credit. The head’s age and property taxes satisfy the senior homeowner pathway and produce $375.
+us,scenario_118,state_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The model incorrectly treated earned income or qualifying dependents as necessary for every refundable state credit. New York’s real property tax credit instead uses the senior homeowner’s age and property-tax burden, yielding $375."
+us,scenario_118,state_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model used an inapplicable 3.5%-of-income excess-tax formula and an unsupported $1,000 maximum. The applicable New York senior real property tax credit computation is capped at $375."
+us,scenario_118,state_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,The answer implies the model omitted the senior homeowner real property tax credit triggered by age 74 and real estate taxes paid. That refundable credit equals $375.
+us,scenario_118,state_refundable_credits,gpt-5.4-nano,llm_error,state_local_rule,False,The model failed to identify the listed age and property taxes as inputs to New York’s refundable real property tax credit. Applying that program yields $375.
+us,scenario_118,state_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,The model incorrectly inferred that Social Security-only income generates no refundable New York credit. The real property tax credit is available independently of earned income and provides this senior homeowner $375.
+us,scenario_118,state_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model selected a $90 household credit instead of applying the refundable senior real property tax credit. The relevant refundable amount is the $375 credit generated by age 74 and $1,633.50 of real estate taxes."
+us,scenario_118,state_refundable_credits,grok-4.3,llm_error,state_local_rule,False,The model overlooked the New York real property tax credit eligibility established by the head’s age and property taxes. The senior homeowner credit is $375.
+us,scenario_118,state_refundable_credits,grok-build-0.1,llm_error,state_local_rule,False,"The model treated the New York EITC and child credits as the only relevant refundable programs. It omitted the real property tax credit, which awards this 74-year-old homeowner $375."
+us,scenario_118,state_refundable_credits,inkling,llm_error,state_local_rule,False,The model incorrectly asserted that no property tax credit applied despite the explicit age and real estate tax facts. Those facts trigger New York’s refundable senior real property tax credit of $375.
+us,scenario_118,state_refundable_credits,kimi-k2.6,llm_error,state_local_rule,False,"The model reviewed credits tied to earnings, children, tuition, and child care but omitted the senior homeowner real property tax credit. The head’s age and real estate taxes produce $375 without any of those other conditions."
+us,scenario_118,state_refundable_credits,minimax-m3,llm_error,state_local_rule,False,The model incorrectly made state tax liability or children necessary to trigger refundable credits. New York’s refundable real property tax credit applies to this qualifying senior homeowner and equals $375.
+us,scenario_118,state_refundable_credits,ox-alpha,llm_error,state_local_rule,False,"The model considered only the EITC and dependent-care credit pathways. It failed to apply the real property tax credit based on age 74 and property taxes paid, which yields $375."
+us,scenario_118,state_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,The model incorrectly equated the absence of taxable or earned income and children with the absence of refundable credits. The senior homeowner real property tax credit uses the listed age and real estate taxes and produces $375.
+us,scenario_118,state_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,The model failed to apply New York’s age-based homeowner real property tax credit despite the head being 74 and paying real estate taxes. The resulting refundable credit is $375.
+us,scenario_119,child1_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model treated 205% FPL as an eligibility ceiling that the household satisfied. Under the applicable Virginia CHIP parameters, the approximately $55,800 household income fails the income test; age 14 is not the disqualifying criterion."
+us,scenario_119,child1_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model incorrectly placed the household income within Virginia's CHIP threshold. Child1 meets the age condition at 14, but the approximately $55,800 household income fails the applicable CHIP income test."
+us,scenario_119,child1_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model misapplied the asserted 205% FPL ceiling and classified the household MAGI as within it. PolicyEngine's applicable income calculation is approximately $55,800 and fails Virginia's CHIP income criterion, while the child's age qualifies."
+us,scenario_119,child1_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly placed household income in a gap above child Medicaid eligibility but below the CHIP ceiling. The child is ineligible for Medicaid and the approximately $55,800 household income also fails Virginia's CHIP income test."
+us,scenario_119,child1_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an estimated Virginia CHIP income range that incorrectly included this household. Although child1 is under 19, the approximately $55,800 household income fails the applicable CHIP income criterion."
+us,scenario_119,child1_chip_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model understated the household's FPL ratio as approximately 192% and compared it with an asserted 200% ceiling. PolicyEngine uses household income of approximately $55,800, which fails the applicable Virginia CHIP income test; age 14 satisfies the age condition."
+us,scenario_119,child1_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly classified the household as below a roughly 200–205% FPL FAMIS ceiling. The approximately $55,800 household income fails Virginia's applicable CHIP income test, and Medicaid ineligibility does not itself create CHIP eligibility."
+us,scenario_119,child1_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model understated the applicable household income as $46,180 instead of the approximately $55,800 used for CHIP eligibility. That erroneous income derivation placed the child inside its asserted 143–200% FPL band; the correct income fails the CHIP test."
+us,scenario_119,child1_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child1_chip_eligible. The required output was 0 because age 14 satisfies the age condition but the approximately $55,800 household income fails Virginia's CHIP income test."
+us,scenario_119,child1_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model understated the applicable household income as $52,569 and therefore placed it inside Virginia's CHIP band. PolicyEngine uses approximately $55,800 for this eligibility determination, which fails the applicable income test."
+us,scenario_119,child1_chip_eligible,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model understated the applicable household income as about $52,569 and treated it as just below a 205% FPL ceiling. Child1 meets the age condition, but PolicyEngine's approximately $55,800 income measure fails Virginia's applicable CHIP income test."
 us,scenario_119,child1_medicaid_eligible,claude-fable-5,llm_error,health_coverage,False,"The model correctly found income above Virginia's child Medicaid limit, then incorrectly treated the higher FAMIS/CHIP threshold as a Medicaid-expansion pathway. FAMIS eligibility is reported through CHIP, while the Medicaid category remains NONE at 1.92 times FPL."
 us,scenario_119,child1_medicaid_eligible,claude-sonnet-5,llm_error,household_unit_or_filing_status,False,"The model used a family of four even though the listed household contains one head and two children, and it merged Virginia Medicaid and FAMIS/CHIP into one 205% FPL threshold. The engine's three-person household calculation produces 1.92 times FPL, above the applicable Medicaid limit, with no Medicaid category."
 us,scenario_119,child1_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that 1.92-times-FPL MAGI falls within Virginia's child Medicaid limit. That income exceeds the applicable Medicaid threshold for a 14-year-old, and no alternative Medicaid pathway applies."
 us,scenario_119,child1_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child1_medicaid_eligible, violating the required output contract."
 us,scenario_119,child1_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model incorrectly treated approximately $55,800 as below Virginia FAMIS’s applicable income cutoff by applying a 205% FPL threshold. Child2 satisfies the under-19 age rule but fails CHIP’s income criterion."
-us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model placed the household income within Virginia’s CHIP threshold without applying the engine’s applicable income limit. At approximately $55,800, child2 fails the income criterion despite satisfying the age criterion."
-us,scenario_119,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model incorrectly classified household MAGI as within a 205% FPL FAMIS limit. The applicable Virginia CHIP income test excludes child2 at the household’s approximately $55,800 income; age 11 is not the failing condition."
-us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model asserted that household income was within the applicable 2026 FAMIS child limit without performing the correct threshold comparison. The income fails Virginia’s CHIP criterion, while child2’s age qualifies."
-us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model relied on an estimated Virginia CHIP income range and incorrectly put the household inside it. The approximately $55,800 income fails the applicable limit even though child2 is under 19."
-us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,thresholds_rates,False,"The model understated the household’s relevant poverty-level ratio as approximately 192% and therefore placed it below a 200% FPL ceiling. Under the engine’s income calculation and applicable Virginia limit, child2 fails CHIP’s income test."
-us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used an unsupported MAGI of $46,180, far below the approximately $55,800 household income used for the CHIP test, and therefore understated the FPL ratio as 168%. That incorrect income calculation placed child2 inside the claimed 143%–200% band when the applicable income test excludes the child."
-us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no child2_chip_eligible value or explanation, so the required output was missing."
-us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used a $52,569 MAGI and incorrectly placed it within Virginia’s CHIP income band. The engine’s approximately $55,800 household income fails the applicable CHIP income limit; child2’s age is qualifying."
+us,scenario_119,child2_chip_eligible,gemini-3.5-flash,llm_error,thresholds_rates,False,"The model applied a 205% FPL ceiling and treated income below that figure as sufficient. Under PolicyEngine’s Virginia 2026 parameters, the approximately $55,800 household income fails CHIP’s income criterion for this 11-year-old."
+us,scenario_119,child2_chip_eligible,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model asserted that household income was within Virginia’s CHIP threshold without applying the engine’s actual 2026 income test. The approximately $55,800 household income is outside the applicable CHIP limit."
+us,scenario_119,child2_chip_eligible,gemini-3.7-flash,llm_error,thresholds_rates,False,"The model used a 205% FPL ceiling and therefore placed the child inside the FAMIS income range. PolicyEngine’s applicable Virginia threshold excludes the household at approximately $55,800."
+us,scenario_119,child2_chip_eligible,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly declared the household within the applicable 2026 FAMIS child income limit. The engine’s income test excludes the household at approximately $55,800, while age 11 satisfies the age condition."
+us,scenario_119,child2_chip_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model substituted an estimated CHIP income range for Virginia’s applicable 2026 parameter. Applying the engine’s threshold to approximately $55,800 makes Child 2 income-ineligible."
+us,scenario_119,child2_chip_eligible,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model understated the household’s poverty ratio as approximately 192% FPL and then compared it with a 200% ceiling. The trace uses household income of approximately $55,800, which fails Virginia’s CHIP income criterion."
+us,scenario_119,child2_chip_eligible,grok-4.6,llm_error,thresholds_rates,False,"The model incorrectly placed household MAGI between the child Medicaid and FAMIS limits using an approximate 200–205% FPL ceiling. Under the engine’s Virginia 2026 income test, approximately $55,800 is outside the CHIP range."
+us,scenario_119,child2_chip_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model reduced MAGI to $46,180 and consequently calculated 168% FPL. The trace uses approximately $55,800 for the relevant household income, which fails the CHIP income test."
+us,scenario_119,child2_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable value or explanation for child2_chip_eligible. The required output is 0 because the 11-year-old fails Virginia’s CHIP income criterion at approximately $55,800 of household income."
+us,scenario_119,child2_chip_eligible,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used MAGI of $52,569 and placed it within Virginia’s CHIP band. The trace’s relevant household income is approximately $55,800, which exceeds the applicable CHIP income limit."
+us,scenario_119,child2_chip_eligible,ox-alpha,llm_error,thresholds_rates,False,"The model treated 205% FPL as the controlling ceiling and concluded that its estimated 204% ratio qualified. PolicyEngine’s applicable Virginia 2026 income test excludes the household at approximately $55,800."
 us,scenario_119,child2_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated that income exceeded the Medicaid limit, then reversed its conclusion by combining Medicaid with CHIP/FAMIS and treating possible CHIP eligibility as Medicaid eligibility. Child2 qualifies through no Medicaid pathway, so proximity to a broader children's coverage threshold cannot produce Medicaid eligibility."
 us,scenario_119,child2_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model conflated Virginia children's Medicaid with FAMIS/CHIP and applied a combined threshold of about 205% FPL to the Medicaid output. At 1.92 times FPL, child2 falls into no Medicaid category; eligibility under a separate FAMIS/CHIP pathway does not make is_medicaid_eligible true."
 us,scenario_119,child2_medicaid_eligible,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model asserted that 1.92-times-FPL MAGI was within Virginia's Medicaid income limit for an 11-year-old. That income exceeds every applicable Medicaid threshold for child2, leaving medicaid_category NONE and Medicaid eligibility false."
 us,scenario_119,child2_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for child2_medicaid_eligible, violating the required output contract."
 us,scenario_119,child2_wic_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model invented an $800 limit on the nonrefundable CTC and treated the refundable CTC allocation as leaving $1,210 of tax. After the $546 CDCC and $200 Saver's Credit, $2,310.23 of nonrefundable CTC offsets all remaining liability."
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status and its standard deduction instead of head-of-household status, then incorrectly denied the CDCC because the head has employer-sponsored health insurance. It also omitted the Saver's Credit and failed to allocate the CTC between its nonrefundable and refundable portions."
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,credit_phaseout,False,"The model incorrectly treated the 14-year-old as eligible only for the $500 Credit for Other Dependents rather than the Child Tax Credit. It also understated the CDCC at $312 instead of $546 and omitted the $200 Saver's Credit, leaving a false residual liability."
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model explicitly found that available nonrefundable CTC could eliminate the remaining tax, then submitted $1,568 without a supporting computation. It also understated CDCC at $312 and omitted the $200 Saver's Credit."
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model arbitrarily limited the nonrefundable CTC to about $800 and omitted both the $546 CDCC and $200 Saver's Credit. The nonrefundable CTC actually used is $2,310.23, exactly exhausting the liability remaining after those credits."
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model applied a hypothetical expiration of the post-TCJA tax structure, including personal exemptions, old brackets, and a $1,000 CTC, instead of the applicable 2026 parameters. It also constructed unsupported medical itemized deductions from premiums and expenses and omitted the Saver's Credit."
-us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model did not actually subtract the available nonrefundable credits from its computed tax, despite acknowledging that the CTC alone was sufficient to cover it. It also omitted the $546 CDCC and $200 Saver's Credit and supplied a residual unsupported by its arithmetic."
-us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used a reverted $1,000-per-child CTC, limiting the total CTC to $2,000. It also understated CDCC at $312 and omitted the $200 Saver's Credit; the applicable nonrefundable CTC is $2,310.23 after the other credits."
-us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model miscomputed AGI as $51,713 instead of $52,568.54 and used an understated head-of-household standard deduction. It then failed to subtract the $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC that eliminate the tax."
-us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $46,180 without support from the listed inputs and applied personal exemptions that do not belong in the reference computation. It also used only $2,000 of CTC, understated CDCC at $312, and omitted the $200 Saver's Credit."
-us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The submitted $6,350 exceeds even the traced $3,056.23 income tax before credits and is incompatible with the model's claim that it subtracted nonrefundable credits. It failed to apply the $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC that reduce the output to zero."
-us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model used an obsolete post-TCJA-expiration structure with personal exemptions and understated the CTC at $2,000. It also understated CDCC at $312 and omitted the $200 Saver's Credit, instead of applying the credits that fully offset $3,056.23 of tax."
-us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated the mortgage balance as supporting a mortgage-interest deduction even though no mortgage interest was listed and unlisted numeric inputs were zero. It also failed to apply the full traced combination of CDCC, Saver's Credit, and nonrefundable CTC that eliminates the liability."
-us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly reverted to personal exemptions and old 10%/15% brackets for 2026 and limited the CTC to $2,000. Although it identified the Saver's Credit, it understated CDCC at $312 and missed the $2,310.23 nonrefundable CTC actually used."
-us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so its response failed the required output contract."
-us,scenario_119,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly eliminated the EITC instead of applying the two-child phaseout, which leaves $764.23. It also forced refundable CTC to the $3,400 per-family cap rather than computing the unused refundable portion as $2,089.77 after nonrefundable credits."
-us,scenario_119,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated this income as exceeding both refundable-credit eligibility ranges. The two-child EITC remains $764.23, and the CTC refundability rules produce $2,089.77 rather than zero."
-us,scenario_119,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated pre-credit income tax and therefore understated the CTC left for refund, calculating about $900 instead of $2,089.77. It also rounded the still-positive two-child EITC down to zero rather than applying the phaseout to obtain $764.23."
-us,scenario_119,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used taxable income of about $48,300 despite AGI near $52,569 and a head-of-household standard deduction, substantially overstating regular tax. That error made it consume the entire CTC nonrefundably, while the correct computation leaves $2,089.77 refundable and also yields $764.23 of EITC."
-us,scenario_119,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly set EITC to zero and treated the refundable CTC cap as the amount paid. The phaseout leaves $764.23 of EITC, while only $2,089.77 of CTC remains refundable after nonrefundable use."
-us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated 2026 EITC parameters and an estimated standard deduction instead of the applicable engine parameters, producing $922 of EITC and only $723 of refundable CTC. The correct component amounts are $764.23 and $2,089.77."
-us,scenario_119,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that regular tax fully absorbs the CTC and that income fully phases out EITC. The correct credit ordering and liability calculation leave $2,089.77 of refundable CTC, and the two-child EITC phaseout leaves $764.23."
-us,scenario_119,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model used the wrong EITC parameters or phaseout base, producing $1,431 instead of $764.23. It also incorrectly asserted that the CTC was fully used nonrefundably, omitting $2,089.77 of refundable CTC."
-us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $46,180 without any listed deductions supporting that figure, thereby inflating EITC to $2,643. It also understated the unused refundable CTC as $427 instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned a round $3,000 refundable CTC without computing its nonrefundable use and omitted EITC entirely. The correct refundable CTC is $2,089.77 and the separate EITC is $764.23."
-us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model overstated EITC as $1,933 and understated refundable CTC as $172. Applying the correct EITC phaseout and CTC liability allocation yields $764.23 and $2,089.77, respectively."
-us,scenario_119,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used unsupported earned income of $45,524, inflating EITC to $1,808. It also treated the CTC as fully nonrefundable, omitting the $2,089.77 refundable portion."
-us,scenario_119,federal_refundable_credits,gemini-3.6-flash,llm_error,credit_phaseout,False,"The submitted total understates the two required components. The applicable computations produce $764.23 of EITC plus $2,089.77 of refundable CTC, not $2,216.39."
-us,scenario_119,federal_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The model named the correct credit categories but miscomputed their combined amount. The EITC is $764.23 and refundable CTC is $2,089.77, totaling $2,854.01 rather than $1,530.82."
-us,scenario_119,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly treated earned income near $51,913 as above the complete two-child EITC phaseout and omitted the resulting $764.23 credit. It also overstated CTC used nonrefundably, leaving only $1,106 instead of $2,089.77 refundable."
-us,scenario_119,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the full $2,000 per child as refundable without first applying CTC against income-tax liability and the refundable limitation. Only $2,089.77 is refundable, and the model also omitted the $764.23 EITC."
-us,scenario_119,federal_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The unexplained $190 reflects neither applicable refundable-credit component. The correct computation yields $764.23 of EITC and $2,089.77 of refundable CTC."
-us,scenario_119,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model used the wrong EITC phaseout result, reporting $1,496 instead of $764.23. It also incorrectly treated the CTC as fully used nonrefundably, omitting $2,089.77 of refundable CTC."
-us,scenario_119,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model substantially overstated both refundable components, assigning about $3,000 of EITC and $2,363.10 of refundable CTC. Their correct amounts are $764.23 and $2,089.77."
-us,scenario_119,federal_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model overstated EITC as $1,276 and understated refundable CTC as $1,656. Correct application of the EITC phaseout and CTC allocation produces $764.23 and $2,089.77."
-us,scenario_119,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The EITC estimate was close but still high at $785 rather than $764.23, and the model understated refundable CTC as $1,656 rather than $2,089.77. Those errors reduce its total below $2,854.01."
-us,scenario_119,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model treated household income as beyond the complete EITC phaseout and overlooked the refundable CTC. The correct calculation retains $764.23 of EITC and $2,089.77 of refundable CTC."
-us,scenario_119,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used incorrect EITC parameters, yielding $1,300 instead of $764.23. It also incorrectly claimed the CTC was fully absorbed nonrefundably, omitting $2,089.77."
-us,scenario_119,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model greatly overstated EITC as $4,109 instead of applying the upper phaseout range to obtain $764.23. It simultaneously understated refundable CTC as $584 instead of $2,089.77."
-us,scenario_119,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The near-total match resulted from offsetting component errors: it overstated EITC as $1,188 and understated refundable CTC as $1,658. The correct components are $764.23 and $2,089.77, so their total is $2,854.01 rather than $2,846."
-us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for federal_refundable_credits, so the required structured answer was missing."
-us,scenario_119,federal_refundable_credits,kimi-k3,llm_error,credit_phaseout,False,"The model correctly found the $764.23 EITC but overstated the amount of CTC used nonrefundably, leaving only $583.72 refundable. The correct refundable CTC is $2,089.77 after applying the tax-liability and nonrefundable-credit computation."
-us,scenario_119,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly treated $55,000 of wages as exceeding the complete two-child EITC phaseout and assumed tax liability absorbs all CTC. The applicable calculations yield $764.23 of EITC and $2,089.77 of refundable CTC."
-us,scenario_119,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model applied married-filing-jointly EITC thresholds even though this taxpayer files as head of household, and it subtracted only the phaseout increment from the maximum as though the phaseout began near $49,000. That produces a grossly inflated EITC; the correct EITC is $764.23, alongside $2,089.77 of refundable CTC."
-us,scenario_119,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated the $300,000 stock balance as disqualifying investment income; EITC uses investment income generated, and only $800 of taxable interest is listed. It therefore omitted the $764.23 EITC and also understated refundable CTC as $565 instead of $2,089.77."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model arbitrarily limited the nonrefundable CTC to about $800 and treated the refundable CTC cap as requiring $1,210 of tax to remain. The credit allocation instead uses $2,310.23 of CTC nonrefundably after the $546 CDCC and $200 Saver's Credit, reducing the $3,056.23 liability to zero."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,household_unit_or_filing_status,False,"The model used single filing status and a single standard deduction even though the unmarried head maintains a home for two qualifying children and files as head of household. It also wrongly denied the CDCC because employer-sponsored health insurance exists, which has no bearing on dependent-care-credit eligibility, and omitted the Saver's Credit."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly treated the 14-year-old as eligible only for the $500 Credit for Other Dependents; both children are under 17 and qualify for the CTC. It also used a $312 CDCC and omitted the $200 Saver's Credit, instead of applying the traced $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,credit_phaseout,False,"The model explicitly found that available nonrefundable CTC could eliminate the remaining liability, then submitted $1,568 without any computation supporting that residual. Applying the $546 CDCC, $200 Saver's Credit, and $2,310.23 nonrefundable CTC exhausts the $3,056.23 tax."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model restricted the nonrefundable CTC to about $800 by conflating the refundable-credit allocation with the amount usable against tax. The nonrefundable CTC used is $2,310.23, after a $546 CDCC and $200 Saver's Credit, which eliminates the entire pre-credit liability."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model abandoned the applicable 2026 parameters and substituted a speculative post-TCJA-expiration regime with pre-TCJA deductions, personal exemptions, brackets, and a $1,000-per-child CTC. The applicable calculation uses the $24,150 head-of-household standard deduction and current credit rules, under which nonrefundable credits total $3,056.23 and eliminate the tax."
+us,scenario_119,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model stated that the $4,000 potential CTC was limited only by the remaining tax but then subtracted roughly $291 and reported $2,977. It failed to allocate $2,310.23 of CTC nonrefundably and also omitted the $546 CDCC and $200 Saver's Credit that together exhaust the liability."
+us,scenario_119,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model used an obsolete $1,000-per-child CTC and an estimated $23,000 standard deduction. It also calculated the CDCC as $312 and omitted the $200 Saver's Credit; the applicable credits are $546 of CDCC, $200 of Saver's Credit, and $2,310.23 of nonrefundable CTC."
+us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model miscomputed AGI as $51,713 rather than $52,568.54 and used an estimated $21,900 head-of-household deduction instead of $24,150. Its explanation then treated $1,395 as bracket tax without subtracting the CDCC, Saver's Credit, and nonrefundable CTC required by the requested after-credit output."
+us,scenario_119,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model invented an AGI of $46,180 and personal exemptions, producing the wrong taxable income and tentative tax. It further used only $2,000 of CTC and a $312 CDCC while omitting the $200 Saver's Credit; the traced nonrefundable credits fully offset $3,056.23."
+us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The submitted $6,350 exceeds the $3,056.23 tax before any credits and has no supporting arithmetic consistent with the listed income and head-of-household standard deduction. The model failed to compute the tax base and failed to subtract the $3,056.23 of usable nonrefundable credits."
+us,scenario_119,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model substituted speculative post-TCJA deductions and personal exemptions, understated regular tax as about $2,755, and used only a $312 CDCC plus $2,000 CTC. It omitted the $200 Saver's Credit and the applicable credit amounts of $546 for CDCC and $2,310.23 for nonrefundable CTC."
+us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model deducted estimated mortgage interest even though no mortgage-interest payment was listed and unlisted numeric inputs were expressly zero. It also failed to show or apply the CDCC, Saver's Credit, and usable nonrefundable CTC that reduce the correctly computed liability to zero."
+us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-expiration system with personal exemptions and a $2,000 total CTC. Although it included the Saver's Credit, it understated the applicable CDCC and nonrefundable CTC: $546 and $2,310.23 respectively, which together with the $200 Saver's Credit eliminate the tax."
+us,scenario_119,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used invented post-TCJA parameters—a $12,250 head-of-household deduction, personal exemptions, and 10%/15% brackets—instead of the applicable $24,150 standard deduction and 10%/12% computation. It then capped CTC at $2,000 and CDCC at $312 rather than using $2,310.23 and $546, leaving a false residual."
+us,scenario_119,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested output, so it did not perform the required computation or satisfy the structured-output contract."
+us,scenario_119,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model used single filing status despite the head-of-household facts, causing the wrong standard deduction, brackets, and tentative tax. It also used a $312 CDCC and omitted the $200 Saver's Credit; under the correct filing status, $546 of CDCC, $200 of Saver's Credit, and $2,310.23 of nonrefundable CTC exhaust the liability."
+us,scenario_119,federal_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model incorrectly set EITC to zero instead of applying the two-child phaseout that leaves $764.23. It also treated the $1,700-per-child ACTC ceiling as the refundable amount, rather than limiting refundable CTC to the $2,089.77 of CTC remaining after nonrefundable use."
+us,scenario_119,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model falsely imposed an income cutoff that eliminated both credits. The two-child EITC phaseout leaves $764.23, and CTC refundability is governed by unused CTC and the ACTC earned-income limit, producing $2,089.77 rather than zero."
+us,scenario_119,federal_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model overstated pre-credit income tax and therefore reduced unused CTC to about $900 instead of the $2,089.77 refundable amount. It also discarded the $764.23 EITC by treating income near the phaseout endpoint as yielding zero."
+us,scenario_119,federal_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used an internally inconsistent taxable-income calculation, claiming roughly $48,300 after a $32,300 deduction from about $55,800 of income, and thereby overstated tax enough to absorb all CTC. Correct tax and credit ordering leave $2,089.77 refundable CTC, while the EITC phaseout leaves $764.23."
+us,scenario_119,federal_refundable_credits,claude-opus-5,llm_error,credit_phaseout,False,"The model incorrectly eliminated the $764.23 EITC and treated the per-child ACTC cap as the amount payable. Refundable CTC is the $2,089.77 unused portion after nonrefundable CTC application, not the aggregate cap of $3,600."
+us,scenario_119,federal_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model used estimated 2025-based deductions and EITC parameters instead of the applicable 2026 parameters, producing an incorrect EITC. It also understated unused CTC: the correct components are $764.23 of EITC and $2,089.77 of refundable CTC."
+us,scenario_119,federal_refundable_credits,claude-sonnet-5,llm_error,credit_phaseout,False,"The model incorrectly concluded that regular tax fully absorbs the CTC and that the two-child EITC is fully phased out. The actual credit ordering leaves $2,089.77 of refundable CTC, and the applicable phaseout leaves a $764.23 EITC."
+us,scenario_119,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model applied incorrect EITC parameters and ignored the earned-income-versus-AGI comparison, overstating EITC as $1,431 instead of $764.23. It also incorrectly declared the CTC fully used nonrefundably, omitting $2,089.77 of refundable CTC."
+us,scenario_119,federal_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI to $46,180 without support from the listed deductible contributions, which drove its EITC estimate up to $2,643. It also overstated nonrefundable CTC use, leaving only $427 refundable instead of $2,089.77."
+us,scenario_119,federal_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,credit_phaseout,False,"The model assigned a round $3,000 refundable CTC without computing how much CTC remained after nonrefundable use or applying the ACTC limitation. That computation yields $2,089.77, and it also omitted the $764.23 EITC."
+us,scenario_119,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model overstated EITC as $1,933 by misapplying the two-child phaseout and understated refundable CTC as $172 by overstating nonrefundable CTC use. The correct components are $764.23 and $2,089.77."
+us,scenario_119,federal_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model used an unsupported earned-income figure of $45,524, causing it to overstate EITC as $1,808 rather than $764.23. It also incorrectly treated all CTC as nonrefundable, omitting the $2,089.77 refundable remainder."
+us,scenario_119,federal_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The submitted total does not equal the traced $764.23 EITC plus $2,089.77 refundable CTC. Its unexplained $2,216.39 therefore reflects an incorrect computation or combination of those two components."
+us,scenario_119,federal_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model named the correct two credit categories but combined them incorrectly. The EITC is $764.23 and refundable CTC is $2,089.77, totaling $2,854.01 rather than $1,530.82."
+us,scenario_119,federal_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model incorrectly set EITC to zero even though the two-child phaseout leaves $764.23. It also overstated the CTC used nonrefundably, leaving $1,106 instead of $2,089.77 refundable."
+us,scenario_119,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated the full $2,000-per-child CTC as automatically refundable and skipped its allocation against income-tax liability. Only $2,089.77 remains refundable, and the model also omitted the $764.23 EITC."
+us,scenario_119,federal_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The unexplained $190 estimate omits most of both traced components. Applying the two-child EITC phaseout and refundable-CTC calculation yields $764.23 and $2,089.77, respectively."
+us,scenario_119,federal_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model used the wrong EITC phaseout result, estimating $1,496 instead of $764.23. It also incorrectly treated the CTC as fully consumed nonrefundably, omitting $2,089.77 of refundable CTC."
+us,scenario_119,federal_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The model substantially overstated EITC as about $3,000 instead of applying the phaseout to obtain $764.23. It also overstated refundable CTC as $2,363.10 rather than the $2,089.77 left after nonrefundable use."
+us,scenario_119,federal_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model overstated EITC as $1,276 instead of $764.23 and understated refundable CTC as $1,656 instead of $2,089.77. Both component calculations therefore used the wrong phaseout or tax-liability allocation."
+us,scenario_119,federal_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model's EITC estimate is close but exceeds the correct $764.23, and its refundable CTC is understated at $1,656. Correctly allocating CTC after nonrefundable credits leaves $2,089.77 refundable."
+us,scenario_119,federal_refundable_credits,grok-4.3,llm_error,credit_phaseout,False,"The model treated the two-child EITC phaseout as complete, but it leaves $764.23 at this income. It also omitted the $2,089.77 refundable portion of the CTC."
+us,scenario_119,federal_refundable_credits,grok-4.5,llm_error,credit_phaseout,False,"The model used incorrect 2026 EITC parameters, producing $1,300 rather than $764.23. It also incorrectly claimed the CTC was fully used nonrefundably, omitting the $2,089.77 refundable remainder."
+us,scenario_119,federal_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model applied incorrect EITC parameters and an obsolete $1,000-per-child CTC amount. The applicable calculations produce $764.23 of EITC and $2,089.77 of refundable CTC."
+us,scenario_119,federal_refundable_credits,grok-build-0.1,llm_error,credit_phaseout,False,"The model overstated EITC as $4,109, which fails to apply the two-child phaseout that reduces it to $764.23. It also understated the refundable CTC as $584 instead of $2,089.77."
+us,scenario_119,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used an incorrect EITC phaseout result of about $1,188 instead of $764.23 and understated refundable CTC as about $1,658 instead of $2,089.77. Its near-total agreement is the result of offsetting component errors."
+us,scenario_119,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or reasoning for federal_refundable_credits, violating the required output contract."
+us,scenario_119,federal_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model correctly identified the $764.23 EITC but overstated nonrefundable CTC use as $3,816.28, leaving only $583.72 refundable. Correct credit ordering and tax liability leave $2,089.77 of refundable CTC."
+us,scenario_119,federal_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly treated $55,000 as above the two-child EITC ceiling and concluded that tax liability absorbs the entire CTC. The phaseout leaves $764.23 of EITC, and $2,089.77 of CTC remains refundable."
+us,scenario_119,federal_refundable_credits,ox-alpha,llm_error,thresholds_rates,False,"The model used an EITC ceiling of roughly $48,000 that does not apply to this 2026 head-of-household case, eliminating a valid $764.23 credit. It also overstated pre-credit tax enough to absorb all $4,400 of CTC, whereas $2,089.77 remains refundable."
+us,scenario_119,federal_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model applied married-filing-jointly EITC thresholds to a head-of-household filer and treated the phaseout-start threshold as though only the excess above it reduced the maximum credit, producing an impossible $6,400 EITC. The correct EITC is $764.23, and refundable CTC is $2,089.77 rather than $1,837."
+us,scenario_119,federal_refundable_credits,qwen3.8-max,llm_error,categorical_eligibility,False,"The model incorrectly treated stock holdings as disqualifying assets for EITC; EITC tests investment income, not the value of stock assets, and only $800 of taxable interest is listed. It therefore omitted the $764.23 EITC and also understated refundable CTC as $565 instead of $2,089.77."
 us,scenario_119,head_chip_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_119,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_119,head_medicare_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -6955,72 +7298,76 @@ us,scenario_119,reduced_price_school_meals_eligible,gemini-3.7-flash,llm_error,t
 us,scenario_119,reduced_price_school_meals_eligible,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model classified the income as below the reduced-price limit without calculating the applicable ratio. The school-meal calculation produces 204% of poverty, which is above the 185% reduced-price threshold."
 us,scenario_119,reduced_price_school_meals_eligible,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used $46,180 of MAGI instead of the $55,800 income base used for school meals. Using the correct income produces a 204% poverty ratio rather than 168%, so the household exceeds the 185% limit."
 us,scenario_119,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model actually reached the correct $39,469 taxable income and approximately $2,012 gross tax, but then discarded that computation and submitted an unsupported $2,296. It also failed to subtract the $152.85 nonrefundable Virginia EITC, which yields $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $4,500 single standard deduction instead of the $8,750 head-of-household deduction and omitted both the $1,560 child-care deduction and $2,790 of personal exemptions. It therefore never derived $39,468.54 of Virginia taxable income or applied the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction, omitted the $1,560 child-care expense deduction, and replaced the actual $152.85 Virginia EITC with an unspecified approximate credit. The required deductions produce $39,468.54 of taxable income and $2,011.94 of gross tax before that EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed approximately $2,116 from its stated inputs and then arbitrarily increased the answer to $2,459 with no tax-law adjustment. It also used an $8,500 standard deduction, omitted the $1,560 child-care deduction, and failed to subtract the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model's stated bracket calculation produces about $2,116, yet it submitted $2,280 and incorrectly described unspecified adjustments as occurring after nonrefundable credits. It also omitted the $1,560 child-care deduction and the $152.85 Virginia EITC and used $8,500 instead of the $8,750 standard deduction."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction and omitted the separate $1,560 Virginia child-care expense deduction. It then applied no nonrefundable credit, missing the $152.85 Virginia EITC that reduces $2,011.94 to $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $2,790 personal exemptions and the $1,560 child-care expense deduction from taxable income. It then invented a $220 adjustment and an approximate child-care credit instead of applying the actual $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only an estimated $8,500 standard deduction, omitting $2,790 of personal exemptions and the $1,560 child-care expense deduction. It also substituted a $124.80 child-care credit for the applicable $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model overstated the Virginia EITC as $660 instead of $152.85 and invented a $78 child-care credit even though the trace treats the $1,560 expense as a deduction. Those mistakes swamp its already understated gross-tax calculation and produce $1,004 instead of $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model used $51,713 as AGI instead of $52,568.54 and did not identify the full $8,750 standard deduction, $1,560 child-care deduction, and $2,790 exemptions. Its answer also omits the final $152.85 nonrefundable Virginia EITC computation."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated AGI by using $46,180 and used an obsolete $3,000 standard deduction instead of $8,750. It also overstated the Virginia EITC as $386.60; the correct credit is $152.85 against $2,011.94 of gross tax."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI at $46,180, used an $8,500 standard deduction, and allowed only $2,400 of exemptions rather than deriving taxable income of $39,468.54. It then used a $62.40 child-care credit instead of the $1,560 child-care deduction and $152.85 Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The submitted amount implies that the model did not follow the traced sequence: $52,568.54 AGI less $8,750, $1,560, and $2,790 yields $39,468.54 taxable income and $2,011.94 gross tax. Subtracting the $152.85 Virginia EITC yields $1,859.09, not $1,696.28."
-us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,credit_phaseout,False,"The unexplained $1,450.48 result does not reconcile to the traced $2,011.94 gross tax less the $152.85 nonrefundable Virginia EITC. It necessarily incorporates an excessive credit or an unsupported additional reduction of taxable income."
-us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model calculated a fictitious $5,669.56 federal EITC and therefore overstated the Virginia EITC as $1,133.91; the potential Virginia EITC is $764.23 and the applicable nonrefundable amount is $152.85. It also omitted the $1,560 child-care deduction and $2,790 of exemptions from its taxable-income calculation."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The model supplied no computation supporting $803.75. The required derivation gives $39,468.54 of taxable income, $2,011.94 of bracket tax, and a $152.85 nonrefundable Virginia EITC, leaving $1,859.09; its number embodies an unsupported $1,208.19 reduction from gross tax."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,credit_phaseout,False,"The model supplied no rule or arithmetic supporting a residual tax of $180. From $2,011.94 of gross Virginia tax, only the traced $152.85 nonrefundable Virginia EITC applies, so the model implicitly invented $1,831.94 of nonrefundable credits."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model claimed that subtracting the standard deduction, exemptions, and child-care deduction left $45,219, but those deductions from $52,568.54 leave $39,468.54. It also used an estimated $267 credit rather than the traced $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The named deductions produce $39,468.54 of taxable income and $2,011.94 of bracket tax, not $1,483.02. The model either embedded an unsupported additional reduction or excessive credit and did not apply the actual $152.85 Virginia EITC as the final step."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model correctly approximated the $2,011.94 bracket tax after the standard deduction, exemptions, and child-care deduction. It then wrongly asserted that no nonrefundable credit applied and failed to subtract the $152.85 Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model invented two $500 Virginia child credits that are absent from the traced calculation. The applicable reduction from $2,011.94 is the $152.85 nonrefundable Virginia EITC, producing $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model gave no taxable-income arithmetic and treated the bracket result itself as the requested post-credit liability. The traced deductions yield $2,011.94 of gross tax, from which the $152.85 Virginia EITC must be subtracted to reach $1,859.09."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 child-care expense deduction, overstating taxable income as $41,279. It also failed to subtract the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used unsupported taxable income of $35,330 instead of the traced $39,468.54. It also stopped at its bracket tax and failed to apply the $152.85 nonrefundable Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model overstated taxable income at about $41,779 by failing to apply the complete $8,750 standard, $1,560 child-care, and $2,790 exemption deductions. It also estimated the Virginia EITC at $238 instead of the traced $152.85."
-us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no value or explanation for the requested variable, so the required output contract was not satisfied."
-us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 child-care expense deduction, overstating taxable income as $41,279. It also incorrectly declared that no nonrefundable credit applied and omitted the $152.85 Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used the wrong Virginia brackets, applying 3% through $17,000 and 5% above that instead of 3% only from $3,000 to $5,000, 5% from $5,000 to $17,000, and 5.75% thereafter. It also used unsupported deduction and exemption amounts and submitted $2,867 despite its own gross-tax calculation being about $2,120."
-us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model invented taxable employer-health-benefit income and treated this unmarried head-of-household unit as married filing jointly with an $18,000 deduction. The correct AGI is $52,568.54, followed by the $8,750 head-of-household deduction, $1,560 child-care deduction, $2,790 exemptions, and $152.85 Virginia EITC."
-us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model used unsupported $50,200 taxable income, omitting the traced $8,750 standard deduction, $1,560 child-care deduction, and full $2,790 exemptions from $52,568.54 of AGI. It also invented a $112 low-income credit instead of applying the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model correctly reconstructed the deductions and $2,011.94 gross tax, then abandoned that computation and submitted an unsupported $2,296. It also failed to subtract the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used a $4,500 single standard deduction instead of the $8,750 head-of-household deduction and omitted the $1,560 child-care deduction and $2,790 of exemptions. It also never applied the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction, omitted the $1,560 child-care expense deduction, and replaced the actual $152.85 Virginia EITC with an unspecified approximation. Those inputs fail to produce the traced $39,468.54 taxable income and $1,859.09 liability."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"The model computed about $2,116 from its stated inputs but arbitrarily submitted $2,459, so its final number contradicts its own arithmetic. It also used an $8,500 standard deduction, omitted the $1,560 child-care deduction, and failed to subtract the $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,other,False,"The model derived approximately $2,116 and then added unexplained adjustments to submit $2,280 even though nonrefundable credits reduce tax. It also used an $8,500 standard deduction, omitted the $1,560 child-care deduction, and did not apply the $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model used a $9,000 standard deduction and omitted Virginia's $1,560 child-care expense deduction. It then treated the gross bracket tax as final and failed to subtract the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted both the $2,790 personal exemptions and the $1,560 child-care expense deduction from taxable income. It then invented $220 and roughly $312 credits instead of applying the traced $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model subtracted only an estimated $8,500 standard deduction, omitting $2,790 of personal exemptions and the $1,560 child-care deduction. It also substituted a $124.80 child-care credit for the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model overstated the Virginia EITC as $660 instead of $152.85 and added an inapplicable $78 child-care credit even though the trace treats the $1,560 expense as a deduction. Its unexplained $34,780 taxable income also differs from the correctly deducted $39,468.54."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The stated $51,713 AGI omits part of the traced $52,568.54 AGI, and the model did not identify the $8,750 standard deduction, $1,560 child-care deduction, or $2,790 exemptions. It also failed to apply the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model understated Virginia AGI as $46,180 and used a $3,000 standard deduction instead of $8,750. It also overstated the nonrefundable Virginia EITC as $386.60 rather than $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model understated AGI as $46,180, used an $8,500 standard deduction, and allowed only $2,400 of exemptions while omitting the $1,560 child-care deduction. It then applied a $62.40 child-care credit instead of the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,other,False,"The submitted $1,696.28 does not follow from the traced $39,468.54 taxable income, $2,011.94 bracket tax, and $152.85 nonrefundable Virginia EITC. The model's generic explanation conceals a $162.81 over-reduction of the correctly computed liability."
+us,scenario_119,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The submitted $1,450.48 does not follow from the traced deductions and credit: $2,011.94 of gross tax less $152.85 equals $1,859.09. Its answer therefore applies $408.61 of unsupported additional tax reduction."
+us,scenario_119,state_income_tax_before_refundable_credits,glm-5.2,llm_error,credit_phaseout,False,"The model radically overstated federal EITC at this income and consequently claimed a $1,133.91 Virginia EITC instead of $152.85. It also omitted the $2,790 exemptions and $1,560 child-care deduction and used an $8,500 rather than $8,750 standard deduction."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,other,False,"The traced deductions produce $2,011.94 of gross Virginia tax and only a $152.85 nonrefundable EITC reduction, not $803.75. The answer implies $1,208.19 of unsupported nonrefundable reductions beyond the deductions it vaguely referenced."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The correct gross tax is $2,011.94 and the applicable nonrefundable Virginia EITC is only $152.85. The $180 answer therefore implies $1,679.09 of nonexistent additional credits or tax reductions."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model's stated $45,219 taxable income fails to subtract the full $8,750 standard deduction, $1,560 child-care deduction, and $2,790 exemptions from $52,568.54. It also overstated the Virginia EITC as $267 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,credit_phaseout,False,"The named deductions produce taxable income of $39,468.54 and gross tax of $2,011.94, so they cannot produce $1,483.02. The answer implicitly subtracts $528.92 rather than the actual $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,credit_phaseout,False,"The model essentially reported the $2,011.94 gross bracket tax after deductions. It incorrectly declared that no nonrefundable low-income credit applies and omitted the $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,credit_phaseout,False,"The model invented two $500 nonrefundable child credits and subtracted them from the gross Virginia tax. The applicable nonrefundable reduction is the $152.85 Virginia EITC, not $1,000 of child credits."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.3,llm_error,other,False,"The terse $1,874 answer does not match the traced calculation of $2,011.94 gross tax less a $152.85 Virginia EITC. It leaves $14.91 too much liability, reflecting an incorrect or rounded credit computation."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 child-care expense deduction. It also stopped at gross bracket tax and failed to subtract the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 child-care deduction, overstating gross tax. It also calculated the Virginia EITC as $223 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used unexplained taxable income of $35,330 instead of the traced $39,468.54. It then reported gross bracket tax without subtracting the $152.85 nonrefundable Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model's $41,779 taxable income omits the full effect of the $8,750 standard deduction, $1,560 child-care deduction, and $2,790 exemptions. It also overstated the Virginia EITC as about $238 instead of $152.85."
+us,scenario_119,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_119,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction instead of $8,750 and omitted the $1,560 child-care expense deduction. It also incorrectly concluded that no nonrefundable state credit applies, omitting the $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The model used incorrect Virginia brackets, applying 3% through $17,000 and 5% above $17,000 instead of 3% from $3,000 to $5,000, 5% from $5,000 to $17,000, and 5.75% thereafter. It also used an incorrect standard deduction and exemption method, produced arithmetic inconsistent with its own $2,120 subtotal, and omitted the $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model used an $8,500 standard deduction rather than $8,750 and omitted the $1,560 child-care expense deduction. It then incorrectly asserted that no nonrefundable credit applies and failed to subtract the $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model invented taxable employer-health benefits and treated the unmarried head-of-household unit as married filing jointly, applying an $18,000 deduction. Employer-sponsored coverage does not create the imputed AGI it added, and the correct calculation uses $52,568.54 AGI, the $8,750 head-of-household deduction, $1,560 child-care deduction, $2,790 exemptions, and a $152.85 Virginia EITC."
+us,scenario_119,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model's $50,200 taxable income omits most of the $8,750 standard deduction, $1,560 child-care deduction, and $2,790 exemptions. Its stated $2,024 base tax is also arithmetically incompatible with $50,200 under Virginia's brackets, and the applicable nonrefundable credit is the $152.85 Virginia EITC rather than an invented $112 credit."
 us,scenario_119,state_refundable_credits,deepseek-v4-pro,llm_error,thresholds_rates,False,"The model incorrectly applied a Virginia refundable EITC in 2026 and used a 20% rate. No refundable Virginia EITC applies for this tax year, so the computed $1,431 federal EITC produces $0 of state refundable credits."
 us,scenario_119,state_refundable_credits,gemini-3.6-flash,llm_error,thresholds_rates,False,"The model extended Virginia's 15%-of-federal-EITC refundable provision into 2026. That refundable provision does not apply in the benchmark year, so the federal EITC generates no Virginia refundable credit."
 us,scenario_119,state_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model incorrectly treated 20% of its estimated $785 federal EITC as a refundable Virginia credit. Virginia provides no applicable refundable EITC in 2026, so this multiplication should not occur."
 us,scenario_119,state_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly applied a 20% refundable Virginia EITC to its $1,300 federal EITC estimate. No such refundable credit applies for Virginia in tax year 2026, leaving state refundable credits at $0."
 us,scenario_119,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model omitted the required state_refundable_credits output entirely. The required value was $0 because no refundable Virginia credit applies to this household in 2026.
 us,scenario_119,state_refundable_credits,kimi-k3,llm_error,thresholds_rates,False,"The model incorrectly calculated a Virginia refundable credit as 20% of a $764.23 federal EITC. Virginia's modeled 2026 rules provide no applicable refundable EITC, so the correct state contribution from the federal EITC is $0."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,thresholds_rates,False,"The model derived taxable income near $200,500 but then applied tax brackets that produced only $30,238, understating regular tax by roughly $10,000. It also included the $1,750 state refund and used a $10,000 SALT cap instead of the traced AGI and $13,907.78 SALT deduction."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated all Social Security as taxable, used a fabricated $30,000 standard deduction instead of the larger $35,716.73 itemized deduction, and used head-of-household rates for a single filer. It then improperly subtracted $11,297 of assumed withholding, which is not a nonrefundable credit and was unlisted."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the state refund, deducted an ineligible $29 IRA contribution, capped SALT at $10,000, and allowed $3,744 of mortgage interest rather than the traced $2,808. It also omitted the $163.40 net investment income tax and did not carry its own stated bracket calculation consistently into its answer."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model included the $1,750 state refund and limited SALT to $10,000, yielding AGI and taxable income above the traced $232,635.12 and $196,918.41. It also omitted the $163.40 net investment income tax."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only $2,012 of SALT and allowed $3,744 of mortgage interest, producing itemized deductions of about $25,920 instead of $35,716.73. It also included the state refund, deducted the IRA contribution, and omitted the $163.40 net investment income tax."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly treated 2026 as a reversion to pre-TCJA rates, personal exemptions, uncapped SALT, and miscellaneous itemized deductions, despite the applicable 2026 parameters. Its final $42,046 also contradicts each of its own successive tax computations and omits the traced $163.40 NIIT calculation."
-us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model reached approximately the traced AGI, it used only about $24,984 of itemized deductions instead of $35,716.73 and then drastically understated the tax on its own $207,651 taxable-income estimate. It failed to use the $13,907.78 SALT deduction and omitted NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model improperly deducted $7,429 of employer-sponsored insurance premiums from wages and applied obsolete personal-exemption and miscellaneous-itemized-deduction rules. It also calculated NIIT on only $1,853 rather than the traced $4,300 investment-income base."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included the state refund, deducted the $29 IRA contribution, allowed $3,744 of mortgage interest, and claimed a miscellaneous employee-expense deduction. Those choices replaced the traced AGI of $232,635.12 and itemized deductions of $35,716.73 with the wrong taxable-income base, and it omitted NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The answer implies an understated tax computation on the applicable $196,918.41 taxable income and omits the $163.40 NIIT. Its generic explanation never identifies the actual $35,716.73 itemized deduction or the applicable 2026 single-filer bracket calculation."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly excluded the $7,429 employer-sponsored insurance premium from W-2 wages and used miscellaneous itemized deductions, driving AGI below the traced $232,635.12. It also calculated only about $87 of NIIT instead of $163.40 on $4,300."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly reduced wages by employer insurance premiums, restored a personal exemption, and deducted unreimbursed employee expenses. These obsolete or unsupported deductions produced $180,066 of taxable income instead of $196,918.41, while its NIIT base was also understated."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored insurance premiums from wages, reducing AGI to $226,985. It also mislabeled the itemized deduction as $32,972 and omitted the $163.40 NIIT, rather than using the traced $35,716.73 deduction and $196,918.41 taxable income."
-us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted amount understates the $39,858.42 regular tax generated by $196,918.41 of taxable income even before adding $163.40 of NIIT. The generic explanation therefore reflects an incorrect bracket computation or taxable-income base and omits NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model used only the explicitly listed $2,012 property tax as SALT and consequently deducted $24,984 rather than the traced $35,716.73. It also included the state refund and omitted $163.40 of NIIT, leaving taxable income far above $196,918.41."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The answer is below the $39,858.42 regular tax due on the traced $196,918.41 taxable income and also excludes $163.40 of NIIT. Its statement that itemized deductions are limited by the standard deduction reverses the rule: the larger $35,716.73 itemized deduction is used."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model understated the tax produced by the applicable single-filer brackets and treated deductions as credit-like items. Mortgage interest and SALT reduce taxable income; they are not nonrefundable credits, and the final computation must also add $163.40 of NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied post-TCJA-sunset rules, including a personal exemption and an itemized deduction around $40,615. The applicable computation has no personal exemption, uses $35,716.73 of itemized deductions, and imposes $163.40 rather than about $70 of NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model applied a charitable-contribution floor incorrectly and arrived at $210,286.16 of taxable income instead of $196,918.41. The deductible charitable amount is $19,000.94 within total itemized deductions of $35,716.73, and $163.40 of NIIT must be added."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model started from overstated AGI of about $234,414 instead of $232,635.12 and calculated only about $70 of NIIT instead of $163.40. It did not reproduce the traced $35,716.73 itemized deduction or $196,918.41 taxable income."
-us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model restored a personal exemption under an inapplicable post-sunset framework and still produced taxable income of about $203,194 rather than $196,918.41. The applicable computation uses $35,716.73 of itemized deductions, no personal exemption, current 2026 rates, and $163.40 of NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,thresholds_rates,False,"The submitted $39,250 is less than the $39,858.42 regular tax alone and therefore reflects an understated bracket calculation or taxable-income base. It also omits the $163.40 NIIT on $4,300 of investment income."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used obsolete post-sunset provisions, deducting unreimbursed employee expenses and a personal exemption while allowing $3,744 of mortgage interest. The applicable itemized deduction is $35,716.73 with $2,808 of mortgage interest, and NIIT is $163.40."
-us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted unreimbursed employee expenses and used $3,744 of mortgage interest, creating a $40,043 itemized deduction instead of $35,716.73. It also included the state refund, applied the wrong post-sunset tax regime, and omitted $163.40 of NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model included the state refund, deducted an obsolete $2,966 miscellaneous employee expense, and used SALT of about $14,212 rather than $13,907.78. These errors produced taxable income of about $194,235 instead of $196,918.41, despite correctly recognizing an NIIT amount near $163."
-us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric output or explanation for the requested variable, so the required structured answer was missing."
-us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no numeric output or explanation for the requested variable, so the required structured answer was missing."
-us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,thresholds_rates,False,"The zero answer omits the entire federal tax computation. Taxable income of $196,918.41 generates $39,858.42 of regular income tax, and $163.40 of NIIT raises the requested amount to $40,021.82."
-us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used the $18,525 standard deduction instead of the larger $35,716.73 itemized deduction and failed to reduce wages for the traditional 401(k) contribution. It also misstated gross income arithmetically and omitted the $163.40 NIIT."
-us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model treated the explicitly single taxpayer as married filing separately and included the tax-exempt pension in taxable pension income. It also improperly combined a standard deduction with charitable itemization, invented a retirement-income credit, and omitted the $163.40 NIIT."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model used the obsolete $10,000 SALT cap and included the $1,750 state refund in income, producing the wrong AGI and itemized deduction. It also misapplied the 2026 brackets and omitted the $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated all Social Security as taxable, used a fictitious $30,000 standard deduction, ignored the $35,716.73 itemized deduction, and used head-of-household rates for a single filer. It then impermissibly subtracted invented withholding from a tax-liability output."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model included the state refund in AGI, imposed the obsolete $10,000 SALT cap, allowed all $3,744 of mortgage interest instead of the debt-limited $2,808, and omitted the 2026 charitable floor. It also failed to add $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model correctly limited mortgage interest to $2,808 but still used the obsolete $10,000 SALT cap, deducted the full charitable contribution, and included the state refund in AGI. Those errors raised taxable income above the correct $196,918.41, and it omitted $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model counted only real-estate tax in SALT instead of the full $13,907.78 deduction and allowed full mortgage interest and charity without their 2026 limitations. Its resulting $208,465 taxable income does not apply the correct $35,716.73 itemized deduction, and it omitted net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly assumed the TCJA regime expired and substituted restored pre-TCJA rates, a personal exemption, and miscellaneous employee-expense deductions. The applicable 2026 computation instead uses the 10/12/22/24 percent schedule on $196,918.41 and adds $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"Although the model reached approximately the correct AGI, it used only $24,984 of itemized deductions by failing to include the full $13,907.78 SALT amount and mishandling the charitable floor. It then substantially undercomputed tax on its own stated $207,651 taxable income and omitted net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted $7,429 of employer-sponsored premiums from wages and applied pre-TCJA personal-exemption and miscellaneous-itemized-deduction rules. It also used the wrong SALT and mortgage amounts and subjected only $1,853, rather than $4,300, to net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model included the state refund in AGI and incorrectly restored miscellaneous employee-expense deductions under a TCJA-sunset theory. It also used full mortgage interest and the full charitable gift rather than the $2,808 interest deduction and $19,000.94 charitable deduction."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The answer implies an incorrect taxable-income or rate computation: the correct itemized deduction is $35,716.73, leaving $196,918.41 taxable income. Tax on that amount is $39,858.42 before adding $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model improperly deducted employer-sponsored insurance from W-2 wages, reducing AGI below $232,635.12, and invoked miscellaneous deductions that do not enter the computation. It also calculated net investment income tax on an understated base instead of applying 3.8 percent to $4,300."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model improperly removed employer health premiums from wages and claimed a personal exemption and miscellaneous employee-expense deduction. Those obsolete deductions drove taxable income to $180,066 instead of $196,918.41, and its $70 net investment income tax used the wrong base."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model wrongly deducted employer-sponsored insurance from wages and used only $32,972 of itemized deductions, omitting the correct $13,907.78 SALT treatment and charitable floor calculation. It also omitted the $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,thresholds_rates,False,"The submitted amount does not result from the traced $196,918.41 taxable income: the regular-tax schedule produces $39,858.42. The answer also fails to include the additional $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model limited SALT to the $2,012 property tax rather than deducting $13,907.78 and failed to apply the charitable-contribution floor, leaving taxable income overstated by more than $12,000. It also omitted $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The answer implies that the model failed to derive the traced $35,716.73 itemized deduction and $196,918.41 taxable income. Applying the 2026 schedule gives $39,858.42 of regular tax, followed by $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model conflated deductions with nonrefundable credits and understated the tax produced by the income and deduction inputs. The correct deduction calculation leaves $196,918.41 taxable, yielding $39,858.42 of regular tax plus $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,thresholds_rates,False,"The model incorrectly applied a post-TCJA-sunset personal exemption and an itemized total containing restored pre-TCJA deductions. It also computed only about $70 of net investment income tax instead of $163.40 on the full $4,300 investment-income base."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model's $210,286.16 taxable income shows that it failed to apply the full $35,716.73 itemized deduction against $232,635.12 of AGI. It also omitted the separately included $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model started from an overstated AGI of roughly $234,414 rather than $232,635.12 and did not reproduce the $35,716.73 itemized deduction. It also used about $70 of net investment income tax instead of $163.40 on $4,300."
+us,scenario_120,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model incorrectly restored a personal exemption and used a post-TCJA-sunset tax regime. The applicable computation has no such exemption, uses $196,918.41 of taxable income under the 2026 schedule, and includes $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The submitted amount does not reflect the complete traced calculation: $35,716.73 of itemized deductions leaves $196,918.41 taxable income and $39,858.42 of regular tax. The model also omitted the $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,thresholds_rates,False,"The model incorrectly restored pre-TCJA rates, a personal exemption, and the miscellaneous deduction for unreimbursed employee expenses. It also used full mortgage interest and the full charitable gift rather than their 2026 limited amounts, and omitted the correct $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,thresholds_rates,False,"The model used restored 15/25/28 percent brackets, a personal exemption, and an overstated itemized deduction under an incorrect TCJA-sunset framework. The correct 2026 schedule applies to $196,918.41 and is followed by $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model added a miscellaneous employee-expense deduction, allowed full mortgage interest and charity, and used an estimated SALT amount instead of the traced $13,907.78. It then applied an incorrect post-TCJA-expiration bracket regime and omitted net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model overstated AGI by including the state refund and added a miscellaneous employee-expense deduction that does not enter the 2026 itemized total. It also used about $14,212 of SALT rather than $13,907.78, producing taxable income below the correct $196,918.41."
+us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no numeric value or explanation for the requested output, so its response could not be parsed or scored substantively."
+us,scenario_120,federal_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no numeric value or explanation for the requested output, so its response could not be parsed or scored substantively."
+us,scenario_120,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer ignores $232,635.12 of AGI and the resulting $196,918.41 of taxable income. That taxable income produces $39,858.42 of regular tax plus $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model included the state refund in AGI and consequently calculated both the charitable floor and SALT deduction from the wrong income base, producing $197,685 rather than $196,918.41 of taxable income. It also treated the result solely as bracket tax and omitted the separately required $163.40 net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction even though the $35,716.73 itemized deduction is larger, and its gross-income arithmetic omitted the wage reduction and above-the-line deductions needed to reach $232,635.12 of AGI. It also omitted $163.40 of net investment income tax."
+us,scenario_120,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model treated the single filer as married filing separately, counted the tax-exempt private pension as taxable, and combined the standard deduction with a charitable itemized deduction. It also invented a retirement-income credit and omitted the $163.40 net investment income tax."
 us,scenario_120,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,federal_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,free_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
@@ -7030,83 +7377,88 @@ us,scenario_120,head_medicare_eligible,kimi-k3,parse_contract_failure,missing_ou
 us,scenario_120,head_medicare_eligible,minimax-m3,llm_error,age_disability,False,"The model returned 0 without applying Medicare's age-based eligibility rule. At age 76, the head exceeds the age-65 threshold, so the eligibility value is 1 regardless of the household's income, assets, or employer-sponsored coverage."
 us,scenario_120,head_wic_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,local_income_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived the exact three components and their correct $13,496.16 total, then submitted the unrelated value $13,736.63. Its numeric output contradicts its completed computation."
-us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model replaced Connecticut's $827.98 paid-leave contribution with an invented negative $48.79 “net from itemizations” adjustment. Itemized deductions do not offset employee payroll tax, and the mandatory state contribution must be added to federal FICA."
-us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model omitted Connecticut's mandatory employee paid-leave contribution. Adding the $827.98 contribution to its correctly calculated $12,668.17 of federal FICA yields $13,496.16."
-us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly identified $12,668 of federal FICA and about $828 of Connecticut paid-leave tax, but then replaced their $13,496.16 sum with $10,680. Its submitted value does not follow from any of the components it calculated."
-us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly asserted that Connecticut has no mandatory employee-side payroll tax. Connecticut's paid-leave contribution adds $827.98 to the $12,668.17 federal FICA total."
-us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 employer-sponsored insurance premium from covered wages for Social Security, Medicare, and Connecticut paid-leave tax. The trace applies all three taxes to the full $165,597 of earnings, producing $13,496.16."
-us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model invoked an Additional Medicare Tax adjustment even though $165,597 is below the $200,000 withholding threshold and submitted $14,197 without a valid component calculation. The applicable components are ordinary Social Security, ordinary Medicare, and Connecticut paid leave, totaling $13,496.16."
-us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model used $158,168 after subtracting the $7,429 employer-sponsored insurance premium as the base for all payroll-tax components. PolicyEngine applies Social Security, Medicare, and Connecticut paid leave to the full $165,597 of covered earnings."
-us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly reduced FICA and Connecticut paid-leave wages by the $7,429 employer-sponsored insurance premium. Applying 6.2%, 1.45%, and 0.5% to the full covered earnings produces $13,496.16."
-us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used only $158,168 as covered wages after deducting the employer-sponsored insurance premium. The full $165,597 is used for Social Security, Medicare, and Connecticut's paid-leave contribution."
-us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,891 answer matches applying Social Security, Medicare, and Connecticut paid leave to approximately $158,168 rather than the full $165,597. The covered-earnings base is $165,597, yielding $13,496.16."
-us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly deducted the traditional 401(k), Roth 401(k), and employer-sponsored insurance amounts from FICA wages, even though 401(k) deferrals remain subject to FICA and the trace uses full earnings. It also omitted Connecticut's $827.98 paid-leave contribution from its final total."
-us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no mandatory employee state payroll tax and also understated federal FICA, which alone equals $12,668.17 on these wages. Connecticut paid leave adds $827.98 for a total of $13,496.16."
-us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model named the correct 6.2% and 1.45% rates but its $6,180 output does not apply them to $165,597. It also omitted Connecticut's $827.98 paid-leave contribution; the three applicable components total $13,496.16."
-us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model overstated Connecticut paid-leave withholding as $922.50 instead of $827.98. Federal Social Security and Medicare were correct, so using the proper state contribution yields $13,496.16."
-us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model incorrectly subtracted the $617 traditional 401(k) deferral from FICA wages; that deferral remains subject to Social Security and Medicare taxes. It also omitted Connecticut's $827.98 mandatory paid-leave contribution.
-us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model calculated only the 6.2% Social Security and 1.45% Medicare components. It omitted Connecticut's mandatory $827.98 employee paid-leave contribution.
-us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Connecticut has no employee state payroll tax. Connecticut's 0.5% paid-leave contribution adds $827.98 to the $12,668.17 federal FICA amount."
-us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,The model stopped after Social Security and Medicare and treated those as the entire employee payroll tax. It omitted the $827.98 Connecticut paid-leave contribution.
-us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no mandatory employee state payroll taxes. The Connecticut paid-leave contribution is $827.98 and raises the total from $12,668.17 to $13,496.16."
-us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, so the required output was missing."
-us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model returned no payroll-tax value or explanation, so the required output was missing."
-us,scenario_120,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model assigned zero payroll tax despite $165,597 of covered employee wages. Social Security, Medicare, and Connecticut paid leave produce $10,267.01, $2,401.16, and $827.98 respectively."
-us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,"The model correctly calculated federal Social Security and Medicare but incorrectly stated that Connecticut has no mandatory employee state payroll tax. Adding the $827.98 paid-leave contribution produces $13,496.16."
-us,scenario_120,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model imposed an incorrect $9,655 cap on employee Social Security tax even though 6.2% applies to all $165,597, producing $10,267.01. It also omitted ordinary Medicare tax of $2,401.16 and Connecticut paid-leave tax of $827.98."
+us,scenario_120,payroll_tax,claude-fable-5,llm_error,other,False,"The model derived the complete $13,496.16 total, including Connecticut's paid-leave contribution, but submitted $13,736.63 instead. Its numeric output contradicts its own final arithmetic by $240.47."
+us,scenario_120,payroll_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model replaced Connecticut's 0.5% paid-leave contribution with an invented negative $48.79 state adjustment. The required state component is positive $827.98, which must be added to $12,668.17 of federal FICA."
+us,scenario_120,payroll_tax,claude-opus-4.8,llm_error,state_local_rule,False,"The model omitted Connecticut's mandatory employee paid-leave contribution after asserting that the state had no employee payroll tax. Adding the $827.98 contribution to its correctly calculated $12,668.17 of federal FICA yields $13,496.16."
+us,scenario_120,payroll_tax,claude-opus-5,llm_error,other,False,"The model correctly identified $12,668 of federal FICA and approximately $828 of Connecticut paid-leave tax, then replaced their approximately $13,496 sum with $10,680 without a valid computation. Its submitted amount contradicts every component in its explanation."
+us,scenario_120,payroll_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model incorrectly stated that Connecticut has no mandatory employee-side state payroll tax. It omitted the $827.98 Connecticut paid-leave contribution from the otherwise correct $12,668.17 federal FICA total."
+us,scenario_120,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model subtracted the $7,429 employer-sponsored insurance premium from covered wages and calculated all three payroll-tax components on $158,168. PolicyEngine's covered-earnings base is the full $165,597, producing $13,496.16 rather than $12,891."
+us,scenario_120,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model's $14,197 output does not follow the stated rates and thresholds: $165,597 is below the $200,000 Additional Medicare Tax threshold. Standard Social Security and Medicare plus the Connecticut paid-leave contribution total $13,496.16, so the answer includes an unsupported extra $700.84."
+us,scenario_120,payroll_tax,gemini-3.1-pro-preview,llm_error,payroll_tax_base,False,"The model calculated FICA and Connecticut paid-leave tax on $158,168 instead of the full $165,597 of covered earnings. The $7,429 insurance-premium input does not reduce the payroll-tax base used in this case."
+us,scenario_120,payroll_tax,gemini-3.5-flash,llm_error,payroll_tax_base,False,"The model incorrectly deducted the $7,429 employer-sponsored insurance premium from gross wages before applying Social Security, Medicare, and Connecticut PFML rates. Applying those taxes to the full $165,597 yields $13,496.16."
+us,scenario_120,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"The model used $158,168 as FICA and Connecticut paid-leave wages, implicitly excluding the $7,429 insurance premium. The covered-earnings base is $165,597 for all three components."
+us,scenario_120,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"Its $12,891 result matches applying the federal FICA and Connecticut paid-leave rates to $158,168, thereby excluding the listed $7,429 employer-sponsored insurance premium. The correct covered-earnings base is the full $165,597."
+us,scenario_120,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model improperly reduced FICA wages for the traditional 401(k), Roth 401(k), and employer-sponsored insurance amounts. These inputs do not reduce the $165,597 covered-earnings base used here, and the model also omitted Connecticut's $827.98 paid-leave contribution."
+us,scenario_120,payroll_tax,gpt-5.4-mini,llm_error,state_local_rule,False,"The model expressly omitted Connecticut's mandatory employee paid-leave contribution and also understated federal FICA: Social Security and Medicare alone equal $12,668.17, not $10,087. Adding the $827.98 state contribution produces $13,496.16."
+us,scenario_120,payroll_tax,gpt-5.4-nano,llm_error,thresholds_rates,False,"The model named the correct 6.2% Social Security and 1.45% Medicare rates but did not apply them correctly: those taxes alone equal $12,668.17 on $165,597, not $6,180. It also omitted Connecticut's $827.98 paid-leave contribution."
+us,scenario_120,payroll_tax,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model correctly calculated federal FICA but overstated Connecticut paid-leave withholding as $922.50. The state contribution is $827.98 on the covered earnings, making total payroll tax $13,496.16."
+us,scenario_120,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,The model incorrectly subtracted the $617 traditional 401(k) deferral from FICA wages even though employee elective deferrals remain subject to Social Security and Medicare taxes. It also omitted Connecticut's $827.98 employee paid-leave contribution.
+us,scenario_120,payroll_tax,grok-4.3,llm_error,state_local_rule,False,The model computed only the 6.2% Social Security and 1.45% Medicare taxes on gross wages. It omitted Connecticut's $827.98 mandatory employee paid-leave contribution.
+us,scenario_120,payroll_tax,grok-4.5,llm_error,state_local_rule,False,"The model incorrectly asserted that Connecticut imposes no employee state payroll tax. The $827.98 Connecticut paid-leave contribution must be added to its approximately $12,668 federal FICA result."
+us,scenario_120,payroll_tax,grok-build-0.1,llm_error,state_local_rule,False,"The model's statement that there are no other employee payroll taxes omitted Connecticut's mandatory paid-leave contribution. That state component is $827.98, bringing the total to $13,496.16."
+us,scenario_120,payroll_tax,inkling,llm_error,state_local_rule,False,The model incorrectly stated that Connecticut has no mandatory employee state payroll taxes. It therefore omitted the $827.98 paid-leave contribution from its federal FICA calculation.
+us,scenario_120,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing."
+us,scenario_120,payroll_tax,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no payroll_tax value or explanation, so the required output was missing."
+us,scenario_120,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The zero answer ignored all payroll taxes on the reported $165,597 of wages. Those wages generate $10,267.01 of Social Security tax, $2,401.16 of Medicare tax, and $827.98 of Connecticut paid-leave tax."
+us,scenario_120,payroll_tax,ox-alpha,llm_error,state_local_rule,False,The model correctly computed federal Social Security and Medicare taxes but incorrectly denied any Connecticut employee payroll tax. It omitted the state's $827.98 paid-leave contribution.
+us,scenario_120,payroll_tax,qwen-3.7-max,llm_error,state_local_rule,False,The model incorrectly stated that Connecticut has no mandatory employee state payroll tax. Its federal FICA total must include the additional $827.98 Connecticut paid-leave contribution.
+us,scenario_120,payroll_tax,qwen3.8-max,llm_error,other,False,"The model reported only an incorrectly capped Social Security amount and omitted Medicare tax and Connecticut paid-leave tax entirely. The wages produce $10,267.01 of Social Security tax, $2,401.16 of Medicare tax, and $827.98 of state paid-leave tax."
 us,scenario_120,reduced_price_school_meals_eligible,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,self_employment_tax,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,snap,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,ssi,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,state_local_rule,False,"The model estimated the Social Security subtraction and Connecticut taxable income instead of applying the prescribed $25,140 adjustment to federal AGI of $232,635.12. It therefore used an incorrect tax base near $205,000 and did not calculate the exact $250 and $180 recapture adjustments."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,state_local_rule,False,"The model invented a Connecticut deduction for federal income tax, itemized expenses, and an age-based deduction that do not reduce this Connecticut tax base. Connecticut AGI is $207,495.12 after the state Social Security subtraction, and the filer owes tax with no personal credit reducing it to zero."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,state_local_rule,False,"The model denied the Connecticut Social Security subtraction because income exceeded $75,000, leaving its tax base roughly $29,388 too high. The applicable adjustment subtracts $25,140, after which the exact bracket tax and $430 of recaptures yield $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,state_local_rule,False,"The model approximated Connecticut AGI and taxable income near $200,000 without deriving the $25,140 Social Security adjustment or the exact recaptures. The correct Connecticut AGI is $207,495.12, and the applicable brackets plus $250 and $180 recaptures produce $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model used an obsolete rate range topping out at 5.5% for income near $198,800. Connecticut applies the 6% bracket to much of the $207,495.12 tax base and the 6.5% bracket above $200,000, followed by $430 of recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,state_local_rule,False,"The model treated income above $100,000 as eliminating all Connecticut Social Security relief and taxed $234,414. The computation requires a $25,140 Social Security subtraction from federal AGI of $232,635.12, reducing Connecticut AGI to $207,495.12 before the rate and recapture calculations."
-us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model deducted Roth contributions and treated Social Security and municipal-interest rules inconsistently, then applied the graduated rates to an unsupported $198,000 estimate. The correct base is $207,495.12 after the $25,140 Social Security subtraction, and the rate schedule plus recaptures yields substantially more than $7,200."
-us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,state_local_rule,False,"The model fully exempted Social Security and reduced Connecticut AGI to $191,370 instead of applying the $25,140 adjustment that produces $207,495.12. It also invented a $1,549.11 personal credit; the personal credit is zero at this income."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,state_local_rule,False,"The model subtracted only 25% of federally taxable Social Security and separately removed the state refund, producing Connecticut AGI of $223,731. The trace applies a $25,140 Social Security adjustment to federal AGI of $232,635.12 and produces $207,495.12 before the exact recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,state_local_rule,False,"The unexplained $7,800 answer omits the derivation from federal AGI to Connecticut AGI and is inconsistent with taxing the resulting $207,495.12 base under Connecticut's single-filer schedule. Applying the schedule and $430 of recaptures yields $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,state_local_rule,False,"The model expressly denied Social Security relief at high income and therefore taxed federal AGI without the required Connecticut adjustment. Connecticut subtracts $25,140 here, reducing the state AGI to $207,495.12."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model began from an incorrect federal AGI of $225,235 and overstated the Social Security subtraction at $26,711. Federal AGI is $232,635.12 and the Connecticut subtraction is $25,140, leaving a $207,495.12 state tax base to which both recaptures apply."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,state_local_rule,False,"The model used Connecticut taxable income of $212,320 rather than the traced $207,495.12 produced by the $25,140 Social Security subtraction. That overstated base leads to its excessive tax despite recognizing recapture."
-us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,state_local_rule,False,"The unexplained $13,125 answer is consistent with taxing a base above the traced Connecticut AGI or omitting the Social Security adjustment. Connecticut AGI is $207,495.12 after the $25,140 subtraction, and the bracket calculation with $430 of recaptures yields $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,state_local_rule,False,"The model fully exempted Social Security, invented a $15,000 Connecticut standard deduction, and applied a $200 property-tax credit. The actual state adjustment is a $25,140 Social Security subtraction, no such standard deduction reduces the traced $207,495.12 base, and no personal credit reduces the liability."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,state_local_rule,False,"The unexplained $4,414 answer omits most of the liability generated by Connecticut AGI of $207,495.12. Applying the single-filer brackets and the $250 and $180 recaptures, with no personal credit, yields $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The answer states that itemized deductions helped determine Connecticut tax, but Connecticut's calculation here proceeds from state-adjusted AGI without those deductions. The $25,140 Social Security subtraction leaves $207,495.12 subject to the Connecticut schedule and recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,state_local_rule,False,"The model came close to the correct tax base but constructed it by adding tax-exempt interest, removing the state refund, and subtracting 75% of federally taxable Social Security. The trace instead starts from $232,635.12 and applies one $25,140 Connecticut Social Security adjustment to reach $207,495.12; its low-rate recapture was also $250, not $500."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,state_local_rule,False,"The model taxed approximately $235,031 rather than applying the Connecticut Social Security subtraction. The required $25,140 adjustment reduces federal AGI of $232,635.12 to Connecticut AGI of $207,495.12, lowering the liability to $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,state_local_rule,False,"The model applied Connecticut rates and recapture to approximately $234,000 of AGI, omitting the $25,140 Social Security subtraction. The proper state base is $207,495.12."
-us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,state_local_rule,False,"The model treated high income as eliminating Connecticut Social Security relief and used roughly $234,414 of Connecticut AGI. The applicable Social Security adjustment is $25,140, producing $207,495.12, and its $10,895 result also fails to include the correct bracket liability and $430 recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,state_local_rule,False,"The unexplained $7,850 answer is inconsistent with Connecticut's tax on the traced $207,495.12 base. The single-filer brackets plus $250 of low-rate recapture and $180 of middle-rate recapture yield $11,917.18."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,state_local_rule,False,"The model removed all federally taxable Social Security and the state refund, reducing Connecticut AGI to $197,049. The correct state adjustment is a $25,140 Social Security subtraction from $232,635.12, producing $207,495.12, and the tax also includes $430 of recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the superseded 3% and 5% lower brackets instead of the 2% and 4.5% rates applicable in 2026. It also used an understated $201,651 base and omitted the exact $250 and $180 recapture adjustments."
-us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,state_local_rule,False,"The model failed to apply the $25,140 Connecticut Social Security subtraction and instead taxed approximately $234,385. It then deducted a $300 property-tax credit even though no personal credit reduces the traced liability at this income."
-us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value for the requested output, so the structured-output contract could not be evaluated substantively."
-us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,"The model supplied no value for the requested output, so the structured-output contract could not be evaluated substantively."
-us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,state_local_rule,False,"The zero answer omits the entire Connecticut liability on $207,495.12 of Connecticut AGI. No personal credit or AMT adjustment eliminates the bracket tax and $430 of recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,state_local_rule,False,"The model fully exempted Social Security, subtracted an already tax-exempt pension, allowed a $15,000 personal exemption despite its phaseout, and used outdated 3% and 5% lower rates. The actual $25,140 Social Security adjustment leaves $207,495.12, the personal credit is zero, and the 2026 schedule includes $430 of recaptures."
-us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,state_local_rule,False,"The model understated Connecticut taxable income at $202,564 and invented a $500 property-tax credit. The traced base is $207,495.12, and no personal credit reduces the liability after the $250 and $180 recaptures."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model approximated the Social Security subtraction instead of applying the $25,140 Connecticut adjustment and also constructed the wrong federal AGI. Those errors prevented it from reaching Connecticut AGI of $207,495.12 and the ensuing $11,917.18 liability."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented Connecticut deductions for federal income tax, itemized expenses, and age that do not eliminate this taxpayer's Connecticut liability. Connecticut instead starts with federal AGI, applies the $25,140 Social Security subtraction, and taxes $207,495.12 with phased-out personal relief."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model denied any Social Security subtraction because income exceeded $75,000, leaving Connecticut AGI around $236,883. The applicable combined-income calculation produces a $25,140 Social Security adjustment and Connecticut AGI of $207,495.12."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,thresholds_rates,False,"The model used an approximate $200,000 tax base and misstated the second Connecticut bracket as 5% rather than 4.5%. The exact base is $207,495.12, followed by $250 and $180 recapture adjustments."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model understated Connecticut's applicable graduated rates by treating the schedule as topping out near 5.5% and used an unsupported $198,800 base. The calculation taxes $207,495.12 through the 6.5% bracket and includes $430 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model treated income above $100,000 as eliminating the Connecticut Social Security subtraction altogether. The combined-income adjustment is $25,140, reducing the engine's $232,635.12 federal AGI to $207,495.12 before the rate and recapture calculation."
+us,scenario_120,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model deducted Roth contributions, omitted or excluded income inconsistently, and described Social Security as largely exempt without calculating the statutory adjustment. The proper Connecticut Social Security subtraction is $25,140 and the resulting Connecticut AGI is $207,495.12."
+us,scenario_120,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model reduced Connecticut AGI to $191,370 by over-exempting Social Security and then invented a $1,549.11 personal tax credit. Connecticut AGI is $207,495.12, and personal credits are fully unavailable at this income."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model subtracted only $8,904 of Social Security and separately removed the state refund, producing an overstated $223,731 Connecticut AGI. The required Connecticut-specific subtraction is $25,140 from federal AGI of $232,635.12."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The unexplained $7,800 answer is inconsistent with taxing Connecticut AGI of $207,495.12 under the single-filer schedule. It reflects an understated tax base, rates, or both and omits the $430 total recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model explicitly denied the Social Security adjustment because of high income. Connecticut applies a $25,140 Social Security subtraction here, so using federal AGI without that adjustment materially overstates the tax base."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model began from the wrong federal AGI of $225,235 and used an overstated $26,711 Social Security exemption. The trace uses federal AGI of $232,635.12 and a $25,140 subtraction, yielding Connecticut AGI of $207,495.12."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model's $212,320 taxable income does not apply the exact $25,140 Social Security adjustment to federal AGI of $232,635.12. The correct Connecticut AGI and tax base is $207,495.12 before the specified recaptures."
+us,scenario_120,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The unexplained $13,125 answer is consistent with taxing an AGI materially above $207,495.12. It failed to incorporate the full $25,140 Connecticut Social Security subtraction and the exact recapture computation."
+us,scenario_120,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model fully exempted Social Security, created a $15,000 standard deduction that Connecticut does not use in this calculation, and applied a $200 property-tax credit despite the income phaseout. The actual Social Security subtraction is $25,140, and no personal or property-tax credit reduces the liability."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,thresholds_rates,False,"The unexplained $4,414 liability is incompatible with Connecticut taxable income of $207,495.12. It reflects a severe understatement of the taxable base or failure to apply the graduated single-filer rates and $430 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The answer invokes itemized deductions even though they do not reduce Connecticut AGI in this computation. Applying the $25,140 Social Security adjustment produces a $207,495.12 tax base, not the substantially smaller base implied by $7,929."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model used an approximate $208,805 Connecticut income because it independently subtracted the refund and 75% of federally taxable Social Security. The engine applies one exact $25,140 Social Security adjustment to $232,635.12, producing $207,495.12, with recaptures of $250 and $180 rather than the model's $500 and $180."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model taxed approximately $235,031 without applying the $25,140 Connecticut Social Security subtraction. The applicable Connecticut AGI is $207,495.12, which materially lowers the graduated-rate tax."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model used approximately $234,000 as Connecticut AGI and therefore omitted the $25,140 Social Security adjustment. Connecticut taxes $207,495.12 here and adds only the specified $430 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model incorrectly declared Social Security relief unavailable and used roughly $234,414 of Connecticut AGI. The correct Connecticut Social Security subtraction is $25,140, producing Connecticut AGI of $207,495.12."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The unexplained $7,850 answer implies a substantially understated Connecticut tax base or excessive deductions. The proper Connecticut AGI is $207,495.12 after the $25,140 Social Security adjustment, and no personal credit reduces the computed tax."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model subtracted all federally taxable Social Security and the state refund, reducing Connecticut AGI to $197,049. Connecticut's applicable Social Security adjustment is $25,140, yielding $207,495.12, and the tax also includes $430 of recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model improperly combined a full Social Security subtraction, a 100% pension exemption, and removal of the state refund to reach $166,833. The trace allows a $25,140 Social Security adjustment and no pension subtraction, leaving Connecticut AGI of $207,495.12."
+us,scenario_120,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,thresholds_rates,False,"The model used the obsolete 3% and 5% lower brackets instead of the 2026 2% and 4.5% brackets and started from an understated $201,651 base. The exact base is $207,495.12, followed by $430 of Connecticut recapture."
+us,scenario_120,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model failed to apply the $25,140 Connecticut Social Security subtraction and instead taxed approximately $234,385. It also applied a $300 property-tax credit even though no personal credit reduces liability at this income."
+us,scenario_120,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_120,state_income_tax_before_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_120,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The zero answer ignores the positive Connecticut tax base of $207,495.12 after the $25,140 Social Security subtraction. The taxpayer has no personal credit capable of eliminating the resulting graduated-rate liability."
+us,scenario_120,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model invented a Connecticut charitable deduction equal to donations above 2% of AGI and omitted the actual $25,140 Social Security adjustment. Connecticut AGI is $207,495.12; federal-style charitable itemization does not produce the model's $218,938 base."
+us,scenario_120,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model fully exempted Social Security, subtracted an already excluded tax-exempt pension, and allowed a $15,000 personal exemption despite its phaseout. Those errors reduced the base to $136,825 instead of the correct $207,495.12."
+us,scenario_120,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model used an understated $202,564 taxable income and then applied a nonexistent $500 property-tax credit. The calculation uses $207,495.12 and no personal credit, with $250 low-tax recapture and $180 middle-tax recapture included."
 us,scenario_120,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,state_refundable_credits,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_120,tanf,kimi-k3,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model reclassified the generic disability-benefits input as SSDI and added $6,522 of purported taxable Social Security benefits to AGI, even though no Social Security income was listed. That inflated AGI and the medical-expense floor, producing $2,001 of taxable income instead of the zero taxable income obtained from the applicable itemized deduction."
-us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used only the standard deduction and ignored the listed medical expenses and health-insurance premiums when comparing standard and itemized deductions. After the 7.5% AGI floor, the applicable itemized medical deduction eliminates taxable income, so applying 10% to $9,915 was the wrong computation."
+us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model reclassified the $13,044 generic disability-benefits input as SSDI and added $6,522 of purportedly taxable Social Security benefits to AGI. Excluding those benefits from AGI allows the medical-expense itemized deduction, after the 7.5% floor, to eliminate taxable income and produce $0 tax."
+us,scenario_121,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model used the standard deduction and ignored the much larger itemized medical deduction generated by the listed $30,000 of medical expenses and deductible premiums. After the 7.5%-of-AGI medical floor, itemized deductions eliminate taxable income, so applying 10% to $9,915 was erroneous."
+us,scenario_121,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model mechanically selected the $16,100 standard deduction without comparing it with the household’s deductible medical expenses. The medical itemized deduction after the 7.5%-of-AGI floor exceeds taxable income, eliminating the $9,565 taxable-income amount on which it calculated $956.50."
 us,scenario_121,federal_refundable_credits,deepseek-v4-pro,llm_error,credit_phaseout,False,"The model set AGI equal to $17,276 by subtracting employer-sponsored insurance premiums from wages and omitted the $13,044 of disability benefits from the EITC phaseout income calculation. Including those benefits puts AGI above the childless EITC limit, reducing the credit to $0 rather than $139."
 us,scenario_121,federal_refundable_credits,gemini-3-flash-preview,llm_error,credit_phaseout,False,"The model incorrectly used $17,276 as both earned income and AGI and therefore calculated a partially phased-out childless EITC. The disability benefits raise AGI beyond the applicable income ceiling, fully phasing the EITC out instead of leaving $127."
 us,scenario_121,federal_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model asserted childless EITC eligibility from $17,276 of earned income without applying the phaseout using AGI that includes the $13,044 of disability benefits. That higher AGI exceeds the childless EITC limit, so no $168 credit remains."
-us,scenario_121,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated disability, medical expenses, and limited assets as jointly sufficient for South Carolina Medicaid without establishing a qualifying pathway. The head receives no SSI, has MAGI income of 1.61 times FPL, and is assigned no Medicaid category."
-us,scenario_121,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated that countable income exceeded the SSI-related limit, then reversed that result by asserting an unsupported disabled or medically needy pathway. Neither disability alone nor the listed medical expenses assign the head a Medicaid category, and the engine assigns NONE."
-us,scenario_121,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model asserted that the wages and disability income fell within South Carolina's ABD income limit after disregards without performing a calculation that supports that assertion. Disability does not itself confer ABD eligibility, the head receives no SSI, and no qualifying Medicaid category is assigned."
-us,scenario_121,head_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model invented eligibility through a working-disabled Medicaid pathway using a 250% FPL test and an unexplained $30,320 income figure. The head's disability and assets do not place them in such a category under the applicable PolicyEngine rules; medicaid_category is NONE."
-us,scenario_121,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the listed disability flag as automatic Medicaid eligibility. South Carolina requires a qualifying categorical pathway, and this 53-year-old head receives no SSI and is assigned medicaid_category NONE."
-us,scenario_121,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model reduced Medicaid eligibility to being disabled with low income and never identified a South Carolina eligibility pathway. The head's MAGI is 1.61 times FPL, SSI receipt is zero, and no Medicaid category applies."
-us,scenario_121,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied a medical-expense spend-down as though the listed expenses automatically created South Carolina medically needy eligibility. Those expenses do not assign a qualifying Medicaid category in this computation, so the head remains in category NONE."
-us,scenario_121,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for head_medicaid_eligible, violating the required submission contract."
-us,scenario_121,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model equated meeting the prompt's disability criterion with qualifying for SSI-related Medicaid. The head receives no SSI, and disability status alone does not satisfy an income-tested or categorical Medicaid pathway; the assigned category is NONE."
+us,scenario_121,head_medicaid_eligible,claude-haiku-4.5,llm_error,categorical_eligibility,False,"The model treated stated disability, medical expenses, and limited assets as sufficient for South Carolina Medicaid without establishing an available disabled or medically needy eligibility category. South Carolina has no expansion pathway for this childless adult, and the head qualifies through no non-MAGI category."
+us,scenario_121,head_medicaid_eligible,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly calculated that countable income exceeds the SSI-related limit, then reversed that result by inventing qualification through a medically needy or unspecified disability pathway. Disability and high medical expenses do not place this head into any South Carolina Medicaid category implemented here."
+us,scenario_121,head_medicaid_eligible,claude-sonnet-5,llm_error,thresholds_rates,False,"The model asserted that the head's income falls within South Carolina's ABD limit after disregards, but wages plus disability income remain above the SSI-related standard even after the standard earned-income disregards. Low assets do not cure failure of the income and categorical requirements."
+us,scenario_121,head_medicaid_eligible,deepseek-v4-pro,llm_error,categorical_eligibility,False,"The model invented eligibility as an SSDI working-disabled individual based on income below 250% FPL. The facts do not establish an applicable South Carolina working-disabled Medicaid pathway, and neither stated disability nor disability-benefit receipt assigns the head a Medicaid category."
+us,scenario_121,head_medicaid_eligible,gpt-5.4-nano,llm_error,categorical_eligibility,False,"The model treated the disability flag as automatic Medicaid eligibility. Disability is only a gateway to further non-MAGI requirements, and this head satisfies none of South Carolina's Medicaid categories."
+us,scenario_121,head_medicaid_eligible,grok-4.3,llm_error,categorical_eligibility,False,"The model reduced eligibility to disability plus low income and never applied South Carolina's categorical pathway rules. At 1.61 times FPL, the head has no MAGI category and no qualifying disability-based category."
+us,scenario_121,head_medicaid_eligible,grok-4.6,llm_error,categorical_eligibility,False,"The model assumed unreimbursed medical expenses automatically create eligibility through a South Carolina medically needy spend-down. The head is not assigned a medically needy or other qualifying category, so those expenses do not generate Medicaid eligibility."
+us,scenario_121,head_medicaid_eligible,grok-build-0.1,llm_error,categorical_eligibility,False,"The model applied a medically needy spend-down without first establishing that the head belongs to a covered medically needy group. Disability, medical expenses, and assets below an asserted limit do not substitute for the missing South Carolina Medicaid category."
+us,scenario_121,head_medicaid_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for head_medicaid_eligible. It therefore failed the required output contract rather than completing the eligibility determination.
+us,scenario_121,head_medicaid_eligible,minimax-m3,llm_error,categorical_eligibility,False,"The model equated meeting a stated disability criterion with qualifying for SSI-related Medicaid. The head receives no SSI, and disability status alone does not satisfy the income and categorical conditions for a South Carolina disabled-adult pathway."
 us,scenario_121,head_medicare_eligible,claude-fable-5,llm_error,age_disability,False,"The model converted unspecified disability benefits into Social Security Disability Insurance and treated their receipt as sufficient for Medicare. The facts establish neither SSDI entitlement nor completion of the required waiting period, so the disabled 53-year-old does not qualify."
 us,scenario_121,head_medicare_eligible,claude-opus-4.7,llm_error,age_disability,False,"The model assumed that the disability benefits were SSDI and therefore triggered Medicare. The prompt supplies only generic disability benefits, while SSDI entitlement and the requisite waiting period are unlisted and therefore false."
 us,scenario_121,head_medicare_eligible,claude-sonnet-4.6,llm_error,age_disability,False,"The model explicitly assumed completion of Medicare's 24-month SSDI waiting period from the instruction that facts remain constant for the full year. Constancy during one tax year does not establish prior SSDI entitlement, and neither SSDI status nor a completed waiting period was listed."
@@ -7127,93 +7479,99 @@ us,scenario_121,payroll_tax,gemini-3.6-flash,llm_error,payroll_tax_base,False,"T
 us,scenario_121,payroll_tax,gemini-3.7-flash,llm_error,payroll_tax_base,False,"The model wrongly treated employer-sponsored insurance premiums as automatically reducing the FICA wage base. The full $25,665 is subject to employee Social Security and Medicare tax."
 us,scenario_121,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model identified the correct full-wage FICA calculation but submitted $1,911, which does not follow from its own formula. Applying 6.2% and 1.45% to $25,665 produces $1,591.23 and $372.14, totaling $1,963.37; South Carolina adds no employee payroll tax here."
 us,scenario_121,reduced_price_school_meals_eligible,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_121,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated disability as waiving the SNAP gross-income test and proceeded to deduct medical expenses from net income. The household instead fails South Carolina's controlling gross-income limit on approximately $39,909, so those deductions never generate eligibility."
-us,scenario_121,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that a household with a disabled member faces no gross-income test and reduced net income to zero with the medical deduction. The household's approximately $39,909 of gross income fails the applicable South Carolina eligibility screen before benefit computation."
-us,scenario_121,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,The model incorrectly applied a disabled-household net-test-only pathway and used medical expenses to infer a near-maximum allotment. The applicable gross-income screen disqualifies this one-person household before deductions or allotment calculations.
-us,scenario_121,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model correctly found that $39,909 exceeds the ordinary gross-income limit but then incorrectly converted disability status into an exemption from that test. Because the household fails the applicable gross-income screen, its medical deductions and asset calculation do not establish SNAP eligibility."
-us,scenario_121,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly treated this 53-year-old disabled adult as eligible under a net-income-only pathway, then used duplicated premiums, medical expenses, and an unlisted mortgage payment to force income toward zero. The gross-income limit controls and is exceeded, so none of those deductions yields a benefit."
-us,scenario_121,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,The model incorrectly said disability waived the gross-income test and then reduced net income to zero with medical and shelter deductions. The household fails the applicable gross-income limit before those deductions are considered for an allotment.
-us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,"The model incorrectly exempted the household from the gross-income test solely because the head is disabled. Approximately $39,909 of gross income exceeds the controlling limit, preventing the medical deduction from creating SNAP eligibility."
-us,scenario_121,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model incorrectly treated disability as a waiver of the SNAP gross-income limit and awarded the maximum benefit after medical deductions. The household is screened out on gross income before a zero-net-income allotment can be calculated.
-us,scenario_121,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model incorrectly bypassed the gross-income screen for a disabled individual and applied earned-income, standard, and medical deductions to reach zero net income. The applicable gross-income limit is exceeded, so the maximum allotment calculation does not apply."
-us,scenario_121,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model incorrectly assumed disability removed the gross-income test and that excess medical expenses could establish eligibility. The household's gross countable income exceeds the applicable limit, yielding no SNAP entitlement."
-us,scenario_121,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model skipped the controlling gross-income eligibility screen and moved directly to deductions and the maximum-allotment formula for a disabled household. Approximately $39,909 of gross income disqualifies the household before medical deductions affect the benefit."
-us,scenario_121,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model treated qualifying medical costs as sufficient to reduce SNAP net income to zero without first enforcing the applicable gross-income limit. The household fails that limit, so it cannot receive the $298 monthly maximum."
-us,scenario_121,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model bypassed the gross-income eligibility screen and used medical deductions to award the maximum contiguous-state allotment. The household's approximately $39,909 of gross income exceeds the controlling limit, making the benefit zero."
-us,scenario_121,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model incorrectly asserted that a disabled head makes only the net-income test applicable. The household fails South Carolina's gross-income threshold before its medical expenses can reduce net income or produce a maximum allotment.
-us,scenario_121,snap,inkling,llm_error,categorical_eligibility,False,"The model went directly from disability and medical deductions to zero net income and the maximum allotment. It omitted the controlling gross-income test, which the household fails on approximately $39,909 of annual income."
-us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no SNAP value or explanation, so it failed the required output contract rather than completing the eligibility computation."
-us,scenario_121,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model incorrectly assumed disability made the uncapped medical deduction sufficient to establish eligibility and focused on a resource test. The household first fails the applicable gross-income limit, so neither its medical deductions nor its resources lead to a benefit."
+us,scenario_121,snap,claude-fable-5,llm_error,categorical_eligibility,False,"The model treated disability as waiving the applicable gross-income test and proceeded directly to a net-income calculation. The household instead fails the South Carolina gross-income threshold on $39,909 of annual countable income, so its medical deduction and near-maximum allotment calculation never apply."
+us,scenario_121,snap,claude-opus-4.7,llm_error,categorical_eligibility,False,"The model incorrectly asserted that a household with a disabled member has no gross-income test. The applicable South Carolina eligibility pathway screens out this one-person household at $39,909 of annual countable income before medical deductions can reduce net income."
+us,scenario_121,snap,claude-opus-4.8,llm_error,categorical_eligibility,False,"The model assumed disability made the household subject only to the net-income test, then used medical expenses to establish eligibility. The household fails the applicable gross-income limit first, so no allotment is calculated."
+us,scenario_121,snap,claude-sonnet-4.6,llm_error,categorical_eligibility,False,"The model explicitly recognized that $39,909 exceeds the gross limit but then created a disability-based exemption and continued to the net-income test. Under the applicable South Carolina pathway, the failed gross-income screen ends eligibility, regardless of the medical deduction or resource calculation."
+us,scenario_121,snap,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model incorrectly applied an elderly-or-disabled exemption from the gross-income test and used medical deductions to force net income toward zero. It also inferred shelter costs from a mortgage balance despite the prompt listing no mortgage payment, but the dispositive error is bypassing the gross-income threshold that makes SNAP zero."
+us,scenario_121,snap,gemini-3-flash-preview,llm_error,categorical_eligibility,False,"The model treated the disabled status as an automatic waiver of the gross-income test. Because $39,909 exceeds the applicable one-person gross-income limit, the medical and shelter deductions never reach the benefit-calculation stage."
+us,scenario_121,snap,gemini-3.1-pro-preview,llm_error,categorical_eligibility,False,The model incorrectly exempted the household from the gross-income screen solely because the head is disabled. The household is screened out on gross income before its medical expenses can reduce SNAP net income.
+us,scenario_121,snap,gemini-3.5-flash,llm_error,categorical_eligibility,False,The model assumed disability waived the applicable gross-income limit and therefore converted the large medical expenses into maximum benefits. The household fails the South Carolina gross-income threshold before deductions are considered.
+us,scenario_121,snap,gemini-3.6-flash,llm_error,categorical_eligibility,False,"The model bypassed the gross-income screen based on disability and applied earned-income, standard, and medical deductions to reach zero net income. Those deductions do not overcome the household's prior failure of the applicable gross-income eligibility test."
+us,scenario_121,snap,gemini-3.7-flash,llm_error,categorical_eligibility,False,"The model treated disability as exempting the household from the SNAP gross-income test. Annual countable income of $39,909 fails that test, so the excess-medical deduction cannot establish eligibility."
+us,scenario_121,snap,gpt-5.5,llm_error,categorical_eligibility,False,"The model jumped directly to deductions and the maximum-allotment formula for a disabled household. It omitted the applicable gross-income screen, which disqualifies this one-person household before net income is computed."
+us,scenario_121,snap,gpt-5.6-sol,llm_error,categorical_eligibility,False,"The model used qualifying medical costs to reduce net income to zero without applying the controlling gross-income threshold. The household's $39,909 of annual countable income makes it ineligible before the medical deduction is used."
+us,scenario_121,snap,gpt-5.6-terra,llm_error,categorical_eligibility,False,"The model assumed the disabled household could reach the allotment calculation after medical deductions reduced net income to zero. It first fails the applicable one-person gross-income limit, so no maximum benefit is payable."
+us,scenario_121,snap,grok-4.6,llm_error,categorical_eligibility,False,"The model expressly stated that the disabled SNAP unit skips the gross-income test. That shortcut is the error: $39,909 exceeds the applicable South Carolina gross-income limit, ending eligibility before the deductions and $292 allotment calculation."
+us,scenario_121,snap,grok-build-0.1,llm_error,categorical_eligibility,False,The model incorrectly made the net-income test the only eligibility test because the head is disabled. The household fails the applicable gross-income test before medical expenses can reduce net income to zero.
+us,scenario_121,snap,inkling,llm_error,categorical_eligibility,False,"The model applied earned-income, standard, and medical deductions without first enforcing the applicable gross-income threshold. Gross annual countable income of $39,909 disqualifies the household, so the maximum-allotment step is never reached."
+us,scenario_121,snap,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no SNAP value or explanation. It therefore failed the required structured-output contract rather than completing a substantive SNAP calculation.
+us,scenario_121,snap,kimi-k3,llm_error,categorical_eligibility,False,"The model treated disability as sufficient to bypass the gross-income screen and focused on medical deductions and resources. The applicable South Carolina gross-income limit disqualifies the household at $39,909 before either net-income deductions or the asset test can produce eligibility."
 us,scenario_121,ssi,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model explicitly dismissed itemizing as less favorable despite $30,000 of medical expenses plus health-insurance premiums; after the medical-expense AGI floor, those deductions exceed the standard deduction and eliminate South Carolina taxable income. It also invented SSDI treatment for the generic disability-benefit input and applied an erroneous flat-rate calculation, but the omitted medical itemization alone separates its $400 answer from zero."
-us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $686 estimate implies that the model retained positive South Carolina taxable income after ordinary state deductions. It failed to apply the household’s itemized medical deduction from $30,000 of medical expenses and qualifying premiums, which reduces taxable income to zero."
-us,scenario_121,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted only a projected standard deduction from wages and taxed the resulting $10,265. It failed to choose the larger itemized deduction generated by the household’s $30,000 of medical expenses and qualifying premiums, which eliminates South Carolina taxable income after the medical-expense floor."
-us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model treated nearly all wages and disability benefits as gross taxable income, subtracted only a standard deduction, and then applied unsupported estimated exemptions or credits. It omitted the much larger itemized medical deduction from $30,000 of medical expenses and qualifying premiums, which reduces South Carolina taxable income to zero."
-us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used wages minus an estimated standard deduction as South Carolina taxable income and ignored the household’s itemized medical expenses. The deductible amount remaining after the AGI floor, including qualifying premiums, exceeds that standard deduction and eliminates taxable income; its stated graduated-rate arithmetic also does not produce the submitted $94.26."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,other,False,"The reasoning itself derives tax of about $4,867 but submits $6,552, so the submitted value does not follow its calculation. It also uses rounded projected brackets and omits the engine's separate $1,518.82 above-the-line deduction, preventing the exact $43,446.18 taxable-income result."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"It taxes only 50% of Social Security instead of the capped 85% taxable amount of $27,200, invents additional deductions for disability, and treats charitable gifts as an AGI reduction while also claiming a standard deduction. Those errors replace the required $43,446.18 taxable income with an unsupported $32,046 figure."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"It omits the temporary $6,000 senior deduction and therefore uses only the $20,200 standard deduction. This leaves taxable income at $50,446 instead of $43,446.18 and overstates the tax."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,other,False,"Its stated inputs produce roughly $5,011 of tax, but it then asserts that unspecified further deductions reduce the result to $1,971. The submission neither applies the traced $27,200 taxable-income deduction and $1,518.82 above-the-line deduction nor follows its own arithmetic."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"It reaches taxable income close to the traced amount but then computes the ordinary-bracket tax as only about $2,336 and taxes qualified dividends at $273 despite describing them as within the 0%-to-15% range. Applying the 2026 bracket calculation to $43,446.18 yields $4,746.66, not $2,609."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"It incorrectly assumes the TCJA individual provisions sunset for 2026, revives unreimbursed-employee deductions and a personal exemption, and substitutes pre-TCJA rates. The 2026 computation instead uses the $20,200 standard deduction plus the $6,000 senior deduction and the applicable 2026 brackets."
-us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"It omits the $1,824 of qualified dividends from both provisional income and AGI, then omits the $6,000 senior deduction. These errors produce $48,622 of taxable income rather than the traced $43,446.18."
-us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"It applies a post-sunset itemized-deduction and personal-exemption regime, including state income tax and miscellaneous employee expenses. The applicable computation instead takes the $20,200 standard deduction, the $6,000 senior deduction, and the separate $1,518.82 above-the-line deduction."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"It uses itemized deductions and a revived personal exemption under an inapplicable post-TCJA-expiration assumption. This produces $47,320 of taxable income instead of $43,446.18 under the $27,200 traced taxable-income deduction."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"It reports internally inconsistent taxable-income arithmetic, subtracting a $29,600 standard deduction and then an additional $10,808 of itemized deductions from an incorrect AGI. A taxpayer cannot combine standard and itemized deductions, and the traced computation uses $27,200 of taxable-income deductions plus $1,518.82 above the line."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"Its broad estimate fails to apply the specific $20,200 standard deduction, $6,000 senior deduction, and $1,518.82 above-the-line deduction. Those steps establish taxable income of $43,446.18 and tax of $4,746.66."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"It revives the miscellaneous itemized deduction for employee expenses and a personal exemption, then uses itemized deductions instead of the applicable enhanced standard and senior deductions. It also rounds the capital loss incorrectly, producing the wrong AGI and taxable income."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although it identifies the correct $70,646 headline AGI, it does not apply the full traced deduction sequence that reduces taxable income to $43,446.18, including the $6,000 senior deduction and $1,518.82 above-the-line deduction. Its $5,994 result therefore taxes too large a base."
-us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The reported answer is $6,712 while its submitted explanation ends with value = 6979, violating the required agreement between the numeric value and explanation. The response therefore fails the output contract independently of its unsupported tax computation."
-us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"It omits qualified dividends from provisional income, understates taxable Social Security as $26,289.75 instead of $27,200, and omits the $6,000 senior deduction. Those mistakes prevent the required $43,446.18 taxable-income calculation."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"It treats the deductions as fully offsetting income even though the traced deduction sequence leaves $43,446.18 of taxable income. Applying the 2026 brackets to that positive base produces $4,746.66."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"It incorrectly treats tax-exempt income and the listed itemized expenses as eliminating federal taxable income. The household instead has $43,446.18 of taxable income after the applicable deductions, and no nonrefundable credit reduces the resulting tax to zero."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"It assumes a post-TCJA sunset, itemizes unreimbursed employee expenses, and revives the personal exemption. The applicable 2026 calculation uses the $20,200 standard deduction plus the $6,000 senior deduction and reaches $43,446.18 of taxable income."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"It substitutes a $1,000 nonitemizer charitable deduction for the traced $1,518.82 above-the-line deduction. That leaves its taxable-income base $518.82 too high and leads to $4,759 instead of $4,746.66."
-us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"It does not calculate the traced $43,446.18 taxable income from the $1,518.82 above-the-line deduction and $27,200 taxable-income deduction. Its rounded deduction estimate leaves too much income subject to the 2026 brackets."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"It incorrectly equates the absence of earned income with income below the filing and taxable-income thresholds. Taxable pensions, IRA distributions, dividends, and taxable Social Security leave $43,446.18 of taxable income after deductions."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"It assumes post-sunset itemization and a personal exemption, including Minnesota income tax and unreimbursed employee expenses as deductions. The applicable calculation instead uses the $20,200 standard deduction, $6,000 senior deduction, and $1,518.82 above-the-line deduction."
-us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"It itemizes state income tax and unreimbursed employee expenses under an inapplicable post-TCJA regime, then reports taxable income inconsistent with subtracting its own deductions from AGI. The traced deduction sequence produces $43,446.18 of taxable income and uses the applicable 2026 rates."
-us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"It revives a personal exemption and miscellaneous employee-expense deduction and adds estimated Minnesota income tax to itemized deductions. The applicable computation instead takes $27,200 of standard and senior deductions plus the traced $1,518.82 above-the-line deduction."
-us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,It supplied no numeric output or explanation for the requested variable. The required federal tax amount was therefore missing.
-us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"It acknowledges positive taxable income but then invents personal exemptions or credits that fully eliminate it. The applicable deductions leave $43,446.18 taxable, and no nonrefundable credit offsets the resulting $4,746.66 tax."
-us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"Its reasoning computes a final tax of $5,544 but submits $9,505, so the answer does not follow its own calculation. It also treats the negative long-term capital gain as positive preferential-rate income and omits the $6,000 senior deduction and traced $1,518.82 above-the-line deduction."
-us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"It excludes taxable Social Security from AGI, invents a $1,875 elderly or disabled credit despite the income limitation, and treats a charitable deduction as a credit after computing tentative tax. It also submits $5,065 even though subtracting its stated credit from its stated $3,933.60 tentative tax cannot produce that amount."
+us,scenario_121,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model explicitly dismissed itemizing as inferior even though the household reports $30,000 of other medical expenses plus health premiums and over-the-counter expenses. Applying the deductible medical expenses above the 7.5%-of-AGI floor eliminates South Carolina taxable income; using a standard deduction and then applying an invented flat 6.2% calculation produced the erroneous $400."
+us,scenario_121,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The $686 estimate implies that the model retained positive South Carolina taxable income after generic deductions. It failed to apply the household's large itemized medical deduction, which reduces South Carolina taxable income to zero before the rate schedule is applied."
+us,scenario_121,state_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model subtracted only a projected standard deduction from wages and taxed the resulting $10,265. It omitted the itemized deduction for unreimbursed medical expenses above 7.5% of AGI, which is larger than the standard deduction and eliminates taxable income."
+us,scenario_121,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model included the disability benefits in gross income, subtracted only a standard deduction, and then inserted an unexplained reduction for exemptions or credits. It failed to compute the available itemized medical deduction from the reported $30,000 of medical expenses and qualifying premiums, which reduces South Carolina taxable income to zero."
+us,scenario_121,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model asserted that no South Carolina subtractions or nonrefundable credits applied and carried an estimated $9,565 of federal taxable income directly into the state rate schedule. It omitted the itemized medical-expense deduction above the 7.5%-of-AGI floor, which eliminates the taxable-income base before any bracket calculation."
+us,scenario_121,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model used wages minus an estimated standard deduction as South Carolina taxable income and ignored the reported medical costs. The allowable itemized medical deduction is larger than the standard deduction and reduces taxable income to zero, so applying graduated rates to $11,065 was the wrong computation step."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,000 nonitemizer charitable deduction, leaving taxable income $1,000 too high at $44,446 instead of $43,446.18. Its submitted $6,552 also contradicts its own completed tax calculation of approximately $4,867."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model taxed only 50% of Social Security instead of the capped 85% amount and invented separate disability additions to the standard deduction. It also treated charitable gifts as an AGI reduction and then submitted $1,065 despite deriving $3,668."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model stopped after the $20,200 standard deduction and omitted both the $6,000 senior deduction and the $1,000 nonitemizer charitable deduction. That left taxable income at $50,446 rather than $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model used approximate deduction amounts and then abandoned its own taxable-income calculation, asserting without computation that deductions reduced taxable income to about $40,000. The correct deductions produce $43,446.18 of taxable income and $4,746.66 of tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model reached approximately the correct taxable-income range but computed ordinary-bracket tax as only $2,336, far below the 10% and 12% tax on $41,622.18 of ordinary taxable income. It also taxed qualified dividends at 15% even though they remain entirely in the 0% band."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly assumed the TCJA provisions expired for 2026, restored miscellaneous itemized deductions and a personal exemption, and applied pre-TCJA brackets. The enacted 2026 calculation instead uses the $20,200 standard deduction, $6,000 senior deduction, and $1,000 nonitemizer charitable deduction."
+us,scenario_122,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $1,824 of qualified dividends from its pre-Social-Security AGI, producing AGI of $68,822 instead of $70,646.18. It also omitted the $6,000 senior deduction and $1,000 nonitemizer charitable deduction."
+us,scenario_122,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored unreimbursed employee expenses, a personal exemption, and pre-TCJA 15% brackets, while also inserting an unlisted state-income-tax deduction. It failed to apply the enacted standard, senior, and nonitemizer charitable deductions that yield taxable income of $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model incorrectly used post-TCJA-expiration itemized-deduction and personal-exemption rules. It omitted the enacted $20,200 standard deduction, $6,000 senior deduction, and $1,000 nonitemizer charitable deduction."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The model reported internally incompatible figures: $71,468 minus $29,600 is not $69,455, and itemized deductions cannot also be subtracted after taking the standard deduction. The correct deduction sequence yields taxable income of $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The answer implies that the model did not apply the full $27,200 deduction package consisting of the standard, senior, and nonitemizer charitable deductions. Those deductions reduce taxable income to $43,446.18 before the qualified-dividend worksheet."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored the miscellaneous itemized deduction for employee expenses and a personal exemption. It should instead use the standard deduction plus the senior and nonitemizer charitable deductions, totaling $27,200."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"Although it identified the relevant income, the submitted tax implies it omitted part of the $27,200 deduction package. Taxable income is $43,446.18, with $1,824 of qualified dividends taxed at 0%."
+us,scenario_122,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,other,False,"The model provided no usable computation and its stated answer of $6,712 conflicts with the $6,979 value embedded in its explanation. Neither figure reflects the $43,446.18 taxable-income calculation and resulting $4,746.66 tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,taxable_income_or_deductions,False,"The model omitted qualified dividends from provisional income, understating taxable Social Security, and used estimated deduction parameters that excluded the $6,000 senior and $1,000 nonitemizer charitable deductions. Taxable Social Security is $27,200 and taxable income is $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that deductions fully offset income. After all applicable deductions, taxable income remains $43,446.18 and produces $4,746.66 of tax."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model treated tax-exempt income and itemized expenses as sufficient to eliminate tax, even though tax-exempt income does not create a deduction and itemizing is inferior here. The applicable deductions leave $43,446.18 taxable."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly applied post-TCJA-sunset itemized deductions, a personal exemption, and 15% ordinary brackets. The 2026 rules instead produce $43,446.18 of taxable income through the standard, senior, and nonitemizer charitable deductions."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,thresholds_rates,False,"The model identified the standard, senior, and $1,000 nonitemizer charitable deductions and reached approximately the correct taxable income. It then used approximate bracket parameters rather than applying 10% through $12,400 and 12% to the remaining $29,222.18 of ordinary taxable income."
+us,scenario_122,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,thresholds_rates,False,"The model identified the relevant deduction categories but did not compute the exact taxable income and bracket tax. The correct ordinary taxable income is $41,622.18, producing $4,746.66 while the qualified dividends are taxed at 0%."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated the absence of earnings as eliminating income tax and asserted that deductions and exemptions reduced income below the filing threshold. Pension, IRA, taxable Social Security, and investment income leave $43,446.18 taxable."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored miscellaneous itemized deductions, a personal exemption, and 15% brackets, and inserted an estimated Minnesota income-tax deduction. It should use the $27,200 standard, senior, and nonitemizer charitable deduction package."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model incorrectly used miscellaneous itemized deductions and a personal exemption under assumed post-sunset rules. The applicable 2026 deductions instead reduce taxable income to $43,446.18."
+us,scenario_122,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored miscellaneous employee-expense deductions and 15% brackets, inserted an estimated state-income-tax deduction, and failed to subtract the personal exemption it referenced when stating taxable income. The enacted deduction pathway yields $43,446.18 of taxable income."
+us,scenario_122,federal_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model incorrectly restored miscellaneous itemized deductions and a personal exemption and inserted an estimated Minnesota tax deduction. It missed the enacted standard, senior, and nonitemizer charitable deduction pathway totaling $27,200."
+us,scenario_122,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no output or explanation for the requested variable.
+us,scenario_122,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model acknowledged positive taxable income but then invented enough personal exemptions or credits to eliminate it. No such offset applies; taxable income remains $43,446.18 and tax is $4,746.66."
+us,scenario_122,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model correctly computed income, the standard deduction, and the $6,000 senior deduction but omitted the $1,000 nonitemizer charitable deduction. That omission left taxable income at $44,446 instead of $43,446.18 and overstated tax by $120."
+us,scenario_122,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,other,False,"The model omitted the $6,000 senior and $1,000 nonitemizer charitable deductions, incorrectly treated the $1,447 capital loss as preferential long-term capital-gain income, and then submitted $9,505 despite deriving $5,544. The correct taxable income is $43,446.18 and only the $1,824 qualified dividend receives preferential treatment."
+us,scenario_122,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model excluded taxable Social Security from AGI, invented a $1,875 elderly/disabled credit, and treated a charitable deduction as a post-tax credit. Taxable Social Security is $27,200, the elderly/disabled credit is eliminated by its income reductions, and the deductions yield $43,446.18 of taxable income."
 us,scenario_122,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_122,head_medicaid_eligible,claude-sonnet-5,llm_error,categorical_eligibility,False,"The model treated being age 73, blind, and disabled as automatic categorical eligibility and conflated Medicare Savings Programs such as QMB and SLMB with full Medicaid eligibility. It also dismissed resource limits despite the disclosed $44,000 in bank assets and $177,000 in stock assets; the head qualified through none of Minnesota’s Medicaid pathways."
 us,scenario_122,head_medicaid_eligible,glm-5.2,llm_error,categorical_eligibility,False,"The model correctly found income far above the 138% FPL MAGI limit, then reversed that result solely because the head was elderly, blind, and disabled. Those characteristics do not automatically confer Medicaid eligibility, and the head qualified through no alternative category."
 us,scenario_122,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"The model correctly identified that no wage earnings were provided and that payroll tax therefore equals zero, then contradicted that reasoning by imposing an unsupported $1,270.64 tax on unspecified ""taxable income."" Retirement and investment income is not an employee Social Security or Medicare tax base."
 us,scenario_122,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no parseable payroll_tax output or explanation, violating the required structured-output contract."
 us,scenario_122,self_employment_tax,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model wrongly phased the Minnesota Social Security subtraction down to little or nothing and omitted the $4,977.18 charitable-contribution subtraction. Those errors left roughly $52,000 taxable instead of $19,219 and drove the tax to $2,896."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model treated age, blindness, the standard deduction, and the Social Security subtraction as eliminating all Minnesota taxable income. The applicable Minnesota adjustments leave $19,219 taxable, so the liability does not floor at zero."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly approached the Social Security and charitable subtractions, then invented an additional senior subtraction or elderly/disabled credit that eliminated the remaining tax. No such step reduces this result to zero; the actual adjustments leave $19,219 taxable."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model subtracted the full $32,000 gross Social Security benefit instead of the $27,200 federally taxable amount used by Minnesota and then invoked unspecified elderly or disability credits. This over-subtraction incorrectly erased the $19,219 taxable base."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $44,000 taxable base because it understated Minnesota’s Social Security and charitable subtractions. The correct subtraction and deduction sequence reduces taxable income to $19,219 before applying the brackets."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $15,000 Minnesota pension/IRA subtraction and treated unreimbursed employee expenses as an itemized deduction, while failing to apply the actual $4,977.18 charitable subtraction. Minnesota’s real adjustment sequence yields $19,219 taxable and $1,028.22 of tax, not $536."
-us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted age/blind deduction effects and invoked an elderly/disability credit to eliminate income after the Social Security subtraction. The valid subtractions and deductions leave $19,219 taxable rather than zero."
-us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model deducted only estimated itemized deductions and a personal exemption from AGI, entirely omitting the $27,200 Minnesota Social Security subtraction and $4,977.18 charitable subtraction. That inflated taxable income from $19,219 to $48,529."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model reduced AGI only by an estimated standard deduction and age/blind additions, leaving $51,946 taxable. It failed to apply Minnesota’s $32,177.18 of state subtractions, chiefly Social Security and charitable contributions, which help reduce taxable income to $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $2,420 reflects a generic AGI-minus-deductions estimate rather than Minnesota’s specific subtraction calculation. Applying the $27,200 Social Security subtraction and $4,977.18 charitable subtraction in the full derivation leaves $19,219 taxable and $1,028.22 of tax."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied only a partial Social Security subtraction and omitted the specific $4,977.18 charitable subtraction. Minnesota’s traced subtractions total $32,177.18 and lead to $19,219 taxable, far below the base implicit in $3,013."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model’s estimated $51,665 taxable income shows that it largely failed to apply Minnesota’s Social Security and charitable subtractions. The correct state subtractions total $32,177.18, and the completed deduction sequence leaves $19,219 taxable."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model stopped at $25,176 after subtracting Social Security and an estimated standard deduction. It omitted the Minnesota charitable-contribution subtraction and used the wrong remaining deduction amount; the correct taxable base is $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model did not provide a reproducible taxable-income calculation and its stated reasoning value of $1,308 also conflicts with its submitted $2,389. Minnesota’s traced adjustments produce $19,219 taxable and bracket tax of $1,028.22."
-us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used non-Minnesota rates of 2.85% and 4.80% and treated $14,926 of itemized deductions as the operative state adjustment. Minnesota’s calculation instead includes the $4,977.18 charitable subtraction, reaches $19,219 taxable, and applies Minnesota’s basic brackets."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model asserted that senior and disability adjustments fully eliminate taxable income. The actual Minnesota subtraction and deduction sequence leaves $19,219 taxable, producing positive tax."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated tax after nonrefundable credits as non-positive without identifying any credit that offsets it. The Minnesota calculation leaves $19,219 taxable and $1,028.22 due before refundable credits."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model came close but used estimated additions, deductions, and a charitable subtraction instead of the traced amounts. The exact Minnesota subtraction total is $32,177.18 and the exact taxable base is $19,219, so applying 5.35% alone to its estimated base overstates the result."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The claimed $49,900 taxable income is incompatible with the model’s assertion that it applied the Social Security subtraction and state deduction. Minnesota’s full subtraction and deduction sequence reduces the base to $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income at $21,055 by adding tax-exempt interest and approximating the deduction and charitable subtraction amounts. The traced calculation yields $19,219 taxable, so its 5.35% multiplication used an overstated base."
-us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The $2,640 answer implies that the model did not fully implement the Minnesota Social Security and charitable subtractions despite naming the former. Those subtractions total $32,177.18, and the complete calculation leaves only $19,219 taxable."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model incorrectly treated age-based exemptions and credits as reducing Minnesota taxable income to zero. The applicable adjustments leave $19,219 taxable and do not eliminate the basic tax."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $19,600 standard deduction and omitted the $4,977.18 Minnesota charitable-contribution subtraction, leaving $23,844 taxable. The correct taxable base is $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model deducted only estimated itemized expenses from federal AGI and omitted Minnesota’s $27,200 Social Security subtraction and $4,977.18 charitable subtraction. This inflated taxable income to $53,623 instead of $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model used a rough $50,000 taxable-income estimate without implementing Minnesota’s specific subtractions. The $27,200 Social Security subtraction and $4,977.18 charitable subtraction, together with the remaining deductions, reduce taxable income to $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no output or explanation for the requested variable, so the answer could not be parsed or evaluated as a substantive calculation."
-us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model used approximate standard-deduction and personal-exemption amounts after the Social Security subtraction and omitted the traced $4,977.18 charitable subtraction. Its approximations produced $18,690 taxable rather than the correct $19,219."
-us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a general age-65 subtraction and concluded that the standard deduction eliminates all income. Minnesota’s actual subtractions and deductions leave $19,219 taxable."
-us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted Minnesota’s $27,200 Social Security subtraction and $4,977.18 charitable subtraction, then improperly used a property-tax refund estimate to reduce this income-tax measure. Property-tax relief is not a nonrefundable offset in this output, and the income-tax calculation uses $19,219 of taxable income."
-us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model began from an unexplained $52,092 AGI rather than $70,646.18 and used an unsupported $30,100 standard deduction. Minnesota’s traced state subtractions and remaining deductions instead produce $19,219 taxable and $1,028.22 of tax."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model treated the Social Security subtraction as largely phased out and used an itemized-deduction estimate, leaving about $52,000 taxable. Minnesota instead subtracts $27,200 of Social Security and $4,977.18 for charitable contributions, leaving $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model concluded that age/blind additions and exclusion of Social Security eliminated all taxable income. The applicable Minnesota subtractions leave $19,219 taxable, not zero, producing $1,028.22 at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model correctly approached the Social Security and charitable subtractions, then invented an additional senior subtraction or elderly/disabled credit that erased the remaining liability. The trace leaves $19,219 taxable after the actual $32,177.18 of subtractions, with no further reduction to zero."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model subtracted gross Social Security rather than the $27,200 federally taxable amount and treated age, blindness, deductions, and unspecified credits as sufficient to eliminate tax. The actual Minnesota subtraction calculation leaves $19,219 taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model left about $44,000 taxable by understating the Social Security and charitable-contribution subtractions. Minnesota applies $32,177.18 of total subtractions to AGI, reducing taxable income to $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,taxable_income_or_deductions,False,"The model invented a $15,000 Minnesota pension/IRA subtraction and rebuilt Minnesota deductions through an inapplicable itemized-deduction calculation. The correct adjustments are dominated by the $27,200 Social Security and $4,977.18 charitable subtractions, yielding $19,219 taxable and $1,028.22 of tax."
+us,scenario_122,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model double-counted the effects of Social Security, itemized or standard deductions, age/blind additions, and an elderly/disabled credit until it reduced taxable income to zero. The engine's applicable Minnesota subtractions leave $19,219 taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,taxable_income_or_deductions,False,"The model used generic itemized deductions and a personal exemption to derive $48,529 of taxable income, omitting Minnesota's $27,200 Social Security subtraction and $4,977.18 charitable subtraction. Those subtractions reduce taxable income to $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,taxable_income_or_deductions,False,"The model only reduced AGI by a standard deduction and age/blind additions, leaving $51,946 taxable. It failed to apply Minnesota's traced $32,177.18 of subtractions, including the Social Security and charitable-contribution subtractions."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,taxable_income_or_deductions,False,"The submitted $2,420 implies a taxable base far above $19,219. The model's generic AGI-minus-deductions shortcut omitted or understated Minnesota's $27,200 Social Security and $4,977.18 charitable-contribution subtractions."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,taxable_income_or_deductions,False,"The model applied only a partial Social Security subtraction and therefore retained too much taxable income. Minnesota subtracts the full $27,200 taxable Social Security amount plus $4,977.18 for charitable contributions, leaving $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,taxable_income_or_deductions,False,"The model left $51,665 taxable because it substantially understated the Minnesota Social Security and charitable-contribution subtractions. The correct taxable income is $19,219 after $32,177.18 of subtractions."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,taxable_income_or_deductions,False,"The model applied the $27,200 Social Security subtraction but replaced the traced charitable-contribution subtraction with an estimated standard deduction, producing $25,176 taxable. Minnesota's complete subtraction calculation yields $19,219 taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,taxable_income_or_deductions,False,"The model did not derive the traced $19,219 taxable-income base from the Minnesota subtractions, and its stated reasoning value of 1,308 also conflicts with its submitted 2,389. Applying $32,177.18 of subtractions to AGI and taxing the remainder at 5.35% yields $1,028.22."
+us,scenario_122,state_income_tax_before_refundable_credits,glm-5.2,llm_error,thresholds_rates,False,"The model used incorrect Minnesota rates of 2.85% and 4.80% and substituted $14,926 of itemized deductions for the traced $4,977.18 charitable subtraction. The $19,219 taxable base is entirely taxed at 5.35%, yielding $1,028.22."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The model incorrectly asserted that senior/disabled-related subtractions and the standard deduction eliminated taxable income. The applicable Minnesota subtractions leave $19,219 taxable and do not reduce the liability to zero."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model incorrectly floored the tax at zero after unspecified deductions and credits. Minnesota's traced calculation leaves $19,219 taxable, producing positive pre-refundable-credit tax of $1,028.22."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,taxable_income_or_deductions,False,"The model identified the relevant Social Security and charitable adjustments but estimated their combined taxable-income effect incorrectly. The exact subtractions total $32,177.18, leaving $19,219 rather than the roughly $20,449 base implied by $1,094."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,taxable_income_or_deductions,False,"The model claimed to apply the Social Security subtraction but still left about $49,900 taxable, which omits most of the traced Minnesota adjustments. Total subtractions of $32,177.18 leave only $19,219 taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,taxable_income_or_deductions,False,"The model combined age/blind deductions, tax-exempt interest, and a charitable subtraction into a $21,055 taxable estimate instead of using the traced $19,219 base. The exact Minnesota subtractions total $32,177.18 and produce $1,028.22 at 5.35%."
+us,scenario_122,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The submitted $2,640 implies that the model retained far too much pension, IRA, investment, or Social Security income in the Minnesota taxable base. It failed to reduce AGI by the traced $32,177.18 of Minnesota subtractions to $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.3,llm_error,taxable_income_or_deductions,False,"The model treated age-based exemptions and other subtractions as eliminating all Minnesota taxable income. They leave $19,219 taxable, so the pre-refundable-credit tax is positive."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.5,llm_error,taxable_income_or_deductions,False,"The model used an estimated $19,600 age/blind standard deduction after the Social Security subtraction and obtained $23,844 taxable. The traced charitable-contribution and other Minnesota subtraction treatment instead produces taxable income of $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-4.6,llm_error,taxable_income_or_deductions,False,"The model approximated the post-subtraction and deduction base as $22,000. The exact Minnesota adjustments reduce taxable income to $19,219, which at 5.35% yields $1,028.22."
+us,scenario_122,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,taxable_income_or_deductions,False,"The model used itemized deductions and a miscellaneous-expense floor while omitting the $27,200 Minnesota Social Security subtraction. It therefore left $53,623 taxable instead of the traced $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,inkling,llm_error,taxable_income_or_deductions,False,"The model estimated taxable income near $50,000 without applying the full Minnesota subtraction calculation. The $27,200 Social Security subtraction and $4,977.18 charitable subtraction are the principal adjustments reducing taxable income to $19,219."
+us,scenario_122,state_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no parseable value or explanation for the requested output.
+us,scenario_122,state_income_tax_before_refundable_credits,kimi-k3,llm_error,taxable_income_or_deductions,False,"The model deducted both an estimated age/blind standard deduction and a personal exemption after the Social Security subtraction, producing $18,690 taxable. Minnesota's traced subtraction structure instead yields $19,219 taxable and $1,028.22 of tax."
+us,scenario_122,state_income_tax_before_refundable_credits,minimax-m3,llm_error,taxable_income_or_deductions,False,"The model invented a senior age-65 subtraction and concluded that the standard deduction erased the remaining income. The actual Minnesota subtractions leave $19,219 taxable rather than zero."
+us,scenario_122,state_income_tax_before_refundable_credits,ox-alpha,llm_error,taxable_income_or_deductions,False,"The model computed federal AGI and taxable Social Security incorrectly, then used a miscellaneous employee-expense itemized deduction to leave $26,079 taxable. Federal AGI is $70,646.18, and Minnesota's $32,177.18 of subtractions produce $19,219 taxable."
+us,scenario_122,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,taxable_income_or_deductions,False,"The model omitted Minnesota's $27,200 Social Security subtraction, treated tax-exempt amounts as additions, and then improperly used a property-tax refund estimate to reduce this pre-refundable-credit output. The correct Minnesota subtractions leave $19,219 taxable before applying the 5.35% rate."
+us,scenario_122,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model started from an incorrect $52,092 AGI and applied an unsupported $30,100 standard deduction, producing $21,992 taxable. The traced calculation starts from $70,646.18 and applies $32,177.18 of Minnesota subtractions to reach $19,219."
 us,scenario_122,state_refundable_credits,claude-fable-5,llm_error,credit_phaseout,False,"The model recognized that household income was about $77,500 but still invented a positive renter credit by subtracting an estimated threshold amount and then adding an unsupported adjustment to reach $2,265. At that household income, the 2026 Minnesota renter-credit schedule yields zero, and the working-family and child credits are also zero."
 us,scenario_122,state_refundable_credits,gemini-3.1-pro-preview,llm_error,credit_phaseout,False,"The model treated payment of rent as sufficient for a Minnesota renter credit and assigned an estimated maximum-like amount without applying the income schedule. Approximately $77,460 of household income phases the renter credit to zero."
 us,scenario_122,state_refundable_credits,gpt-5.5,llm_error,credit_phaseout,False,"The model applied a supposed senior/disabled renter-credit schedule that left a $1,070 refund after a high-income copayment. Minnesota’s 2026 income phaseout yields zero at this household income, and senior or disabled status does not preserve a positive credit."
@@ -7227,40 +7585,43 @@ us,scenario_123,child1_head_start_eligible,glm-5.2,parse_contract_failure,missin
 us,scenario_123,child1_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,child1_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,child1_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model omitted the $2,000 non-itemizer charitable deduction and then inconsistently submitted $5,860 despite calculating several other totals. The requested tax-unit output excludes the child's separate return and equals $7,400.24 minus the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $29,200 standard deduction, invented mortgage interest from the loan balance, and claimed two CTCs despite only one child. The applicable deductions are the $32,200 joint standard deduction and $2,000 non-itemizer charitable deduction, with one $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model approximated the joint standard deduction as $31,500 and omitted the $2,000 non-itemizer charitable deduction. Those deductions reduce taxable income to $65,802 before applying the exact 2026 brackets and $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $29,400 of mortgage interest from a mortgage balance even though unlisted interest expense is zero, and it denied the CTC based solely on the child's earnings. The child remains the qualifying age-16 dependent, and the joint return uses the standard deduction, the non-itemizer charitable deduction, and the $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,thresholds_rates,False,"The model identified the correct AGI, standard deduction, and $2,200 CTC but approximated the 2026 bracket calculation. It also omitted the $2,000 non-itemizer charitable deduction, which makes taxable income $65,802 and regular tax $7,400.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model ultimately used projected rather than applicable 2026 deduction and bracket parameters, taking about $30,750 as the standard deduction and a $2,000 CTC. The correct parameters are a $32,200 standard deduction, an additional $2,000 non-itemizer charitable deduction, and a $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model said the $32,200 standard deduction exceeded $18,557 of itemized deductions but nevertheless subtracted the itemized amount, and its submitted $15,300 contradicts its own $7,300 estimate. The return instead deducts $32,200 plus the $2,000 non-itemizer charitable amount."
-us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model applied expired pre-TCJA deductions, exemptions, rates, and a $1,000 CTC, then added the child's separate-return tax to the requested joint tax-unit output. The child's wages and separate tax are excluded, while the parents receive the 2026 deductions and $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model applied expired pre-TCJA rules, invented mortgage interest and state taxes, and added the child's separate-return liability. The requested joint-unit amount excludes that return and uses the $32,200 standard deduction, $2,000 charitable deduction, and $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model treated the child's $45,000 wages as income on the parents' joint return, producing an erroneous $145,002 AGI. The joint tax unit has $100,002 of AGI because the child's earned income belongs on the child's separate return."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model both described $145,002 as joint AGI and added a separate tax on the same child's wages, double counting that income. The requested output covers the parents' $100,002 joint AGI only and excludes the child's return."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model applied expired pre-TCJA exemptions, rates, and a $1,000 CTC, then added the child's separate-return tax. The benchmark variable is the parents' joint tax-unit liability and uses the applicable 2026 deductions and $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in joint AGI, applied personal exemptions, and incorrectly phased out the CTC. Joint AGI is $100,002, and the full $2,200 CTC applies at that income."
-us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model included the child's wages in the parents' AGI and applied expired personal exemptions. The parents' joint AGI is $100,002, with the child's earnings excluded from this tax-unit output."
-us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added the child's separate-return liability to the parents' tax even though the requested PolicyEngine variable is for the joint tax unit. Its parent calculation also used approximate deduction, bracket, and $2,000 CTC parameters instead of the exact 2026 values."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model put the child's $45,000 wages on the married joint return, inflating AGI from $100,002 to $145,002. Those wages belong to the child's separate tax unit and are excluded from this output."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,household_unit_or_filing_status,False,"The submitted $14,677 is consistent with taxing household-wide wages rather than the parents' $100,002 joint AGI and applying the traced deductions and credit. The requested joint-unit derivation yields $65,802 of taxable income and $5,200.24 after the CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in joint AGI. The child's wages are excluded from the parents' return, leaving $100,002 of AGI before the $34,200 total deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model correctly recognized both the $32,200 standard deduction and $2,000 charitable deduction but applied them to $145,002, improperly including the child's wages. Applying them to the parents' $100,002 AGI gives $65,802 of taxable income."
-us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model's regular-tax figure reflects inclusion of the child's wages in the joint tax base and use of a $30,000 standard deduction. The child is a separate tax unit, and the parents use a $32,200 standard deduction plus the $2,000 charitable deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model combined the head's and child's wages in the joint tax base and also invoked unlisted mortgage interest. The parents' joint AGI is $100,002, mortgage interest is zero, and the standard-deduction pathway controls."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied expired pre-TCJA exemptions, brackets, and a $1,000 CTC, then added the child's separate-return tax. The requested amount is confined to the parents' joint unit under the applicable 2026 rules."
-us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model used expired personal exemptions, 10%/15% brackets, and a $1,000 CTC and then added the child's separate tax. The joint-unit output excludes the child's return and applies the current $34,200 deduction and $2,200 CTC."
-us,scenario_123,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model effectively reached the correct parents' taxable-income pathway but then added roughly $3,225 of tax from the child's separate return. The requested variable contains only the parents' joint tax-unit liability."
-us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no numeric output or explanation for the requested variable, so its response could not be evaluated as a substantive tax calculation."
-us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 wages in joint AGI and then invented a $12,000 qualified-overtime deduction. The joint return excludes the child's wages, and the traced deduction is $34,200 from the standard deduction plus the non-itemizer charitable deduction."
-us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model treated refundable credits as offsets in a quantity explicitly measured before refundable credits and asserted that the CTC erased all liability. Regular tax is $7,400.24, and the usable $2,200 nonrefundable CTC leaves $5,200.24."
-us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the child's wages in joint AGI and invented $27,300 of mortgage interest from the balance and an assumed rate. The joint AGI is $100,002, unlisted mortgage interest is zero, and the standard-deduction pathway applies."
-us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model improperly combined standard and itemized deductions, invented mortgage interest, allowed a QBI deduction for W-2 wages, and created a CDCC without care expenses. Only the $32,200 standard deduction and $2,000 non-itemizer charitable deduction reduce the parents' AGI, and only the $2,200 CTC offsets regular tax."
-us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model awarded a $1,600 refundable CTC without applying the 2026 joint-income phaseout that reduces the household’s CTC to zero. It also invented a residual $205 EITC even though approximately $145,002 of joint income is above the EITC phaseout range for one qualifying child."
-us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model treated one child under 17 and sufficient earned income as enough to produce a $1,500 Additional Child Tax Credit. It omitted the 2026 CTC income phaseout, which eliminates the credit at this household’s approximately $145,002 joint income before any refundable amount remains."
-us,scenario_123,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model used a $1,700 enhanced Additional Child Tax Credit cap and treated sufficient earnings as guaranteeing the full refund. It failed to apply the 2026 CTC income phaseout, which reduces the parents’ credit to zero at approximately $145,002 of joint income."
-us,scenario_123,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied no value or explanation for federal_refundable_credits, violating the required structured-output contract."
-us,scenario_123,federal_refundable_credits,qwen3.8-max,llm_error,credit_phaseout,False,"The model incorrectly treated the child’s $45,000 of wages as making a $2,000 Additional Child Tax Credit fully refundable and used an inapplicable enhanced cap. Refundability does not bypass the 2026 CTC joint-income phaseout, which eliminates the household’s credit at approximately $145,002 of income."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-fable-5,llm_error,taxable_income_or_deductions,False,"The model omitted the $2,000 charitable deduction available in addition to the $32,200 standard deduction, leaving taxable income $2,000 too high. Its submitted $5,860 also contradicts every total in its own reasoning, including its stated parent-only result of $5,440."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model used an obsolete $29,200 standard deduction, invented mortgage interest from the loan balance, and incorrectly claimed credits for two children when only one child exists. It failed to compute the trace’s $34,200 total deduction and single $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,taxable_income_or_deductions,False,"The model used an estimated $31,500 standard deduction and omitted the additional $2,000 non-itemizer charitable deduction. It also used a $2,000 CTC instead of the applicable $2,200 credit."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,taxable_income_or_deductions,False,"The model invented $29,400 of mortgage interest from the mortgage balance and denied the CTC merely because the 16-year-old earned wages. The computation uses the listed deductions only and allows the full $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-opus-5,llm_error,taxable_income_or_deductions,False,"The model stopped after subtracting the $32,200 standard deduction and omitted the separate $2,000 non-itemizer charitable deduction. That omission raised taxable income from $65,802 to $67,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model spent most of its analysis applying an assumed TCJA sunset and ultimately used estimated 2026 parameters rather than the applicable enacted values. Even in its extended-law calculation it omitted the $2,000 non-itemizer charitable deduction and used a $2,000 rather than $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,taxable_income_or_deductions,False,"The model explicitly recognized that the $32,200 standard deduction exceeded $18,557 of itemized deductions, then subtracted the itemized amount anyway. It also omitted the separate $2,000 non-itemizer charitable deduction, and its submitted $15,300 contradicts its stated $7,300 estimate."
+us,scenario_123,federal_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model applied obsolete pre-TCJA deductions, exemptions, brackets, and CTC, then added the child’s separate-return tax to the parents’ requested tax-unit output. The child’s $45,000 wages and separate liability are not part of this variable."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model applied assumed post-sunset rules, invented mortgage interest and state taxes, and added the child’s separate-return tax. The requested value covers the parents’ joint tax unit and excludes the child’s return."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in joint AGI, producing $145,002 instead of $100,002. A dependent child’s earned income is not reported on the parents’ joint return."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model mixed the child’s wages into a stated $145,002 household AGI and also added a separate child-return tax to the output. It further applied obsolete personal exemptions and a $1,000 CTC instead of the 2026 computation."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model applied obsolete post-sunset exemptions, brackets, and a $1,000 CTC, then added the child’s separate-return liability. This output is limited to the parents’ joint tax unit."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model included the child’s wages in joint AGI, used obsolete personal exemptions, and incorrectly phased out the CTC at $145,002. Joint AGI is $100,002 and the full $2,200 CTC applies."
+us,scenario_123,federal_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in the parents’ AGI and applied obsolete personal exemptions. The parents’ joint AGI is $100,002, followed by $34,200 of deductions."
+us,scenario_123,federal_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model added $3,312.08 of tax from the child’s separate return to the parents’ requested tax-unit output. It also omitted the $2,000 non-itemizer charitable deduction from the parents’ taxable-income calculation."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in the married couple’s joint income. The child’s earned income is excluded from the parents’ return, leaving joint AGI of $100,002."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,other,False,"The submitted $14,677 does not follow from the applicable $100,002 joint AGI, $34,200 deduction, 2026 brackets, and $2,200 CTC. Its generic explanation never performed the required tax-unit, deduction, or credit calculation."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in joint AGI, using $145,002 instead of $100,002. Although it identified the standard deduction and CTC amounts, it also omitted the separate $2,000 charitable deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model correctly included the $32,200 standard deduction, $2,000 charitable deduction, and $2,200 CTC but incorrectly included the child’s $45,000 wages in joint AGI. Removing those wages changes taxable income from $110,802 to $65,802."
+us,scenario_123,federal_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,taxable_income_or_deductions,False,"The model used a $30,000 standard deduction rather than the applicable $32,200 amount and did not account for the separate $2,000 non-itemizer charitable deduction. Its gross-tax figure therefore rests on the wrong taxable-income base."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model included the child’s wages in the parents’ income and invoked mortgage interest despite no mortgage-interest payment being listed. Joint AGI is $100,002, and only the trace-supported $34,200 deductions apply."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model applied obsolete personal exemptions, restored brackets, and a $1,000 CTC, then added the child’s separate-return liability. The requested parents’ tax-unit output excludes that child-return tax and uses the enacted 2026 parameters."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"The model included the child’s wages in joint AGI, applied obsolete personal exemptions and brackets, and phased out the CTC under an obsolete $110,000 threshold. The parents’ AGI is $100,002 and receives the full $2,200 CTC."
+us,scenario_123,federal_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model applied obsolete exemptions, brackets, and a $1,000 CTC and then added the child’s separate-return tax. The requested output is the parents’ joint tax-unit liability only."
+us,scenario_123,federal_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model substantially reproduced the parents’ calculation but then added the child’s separate-return tax to the requested output. The $2,000 charitable deduction reduces taxable income rather than AGI, but that labeling error did not change its stated $65,802 taxable income."
+us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no numeric output or explanation for the requested variable.
+us,scenario_123,federal_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in joint income and then applied an overtime deduction to that inflated base. The parents’ joint AGI is $100,002, and the trace contains no overtime deduction."
+us,scenario_123,federal_income_tax_before_refundable_credits,minimax-m3,llm_error,credit_phaseout,False,"The model incorrectly used refundable credits to reduce a quantity explicitly measured before refundable credits and asserted that the CTC erased the liability. The $7,400.24 pre-credit tax exceeds the $2,200 nonrefundable CTC, leaving $5,200.24."
+us,scenario_123,federal_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 wages in joint AGI and applied a $12,500 overtime deduction absent from the computation trace. The correct parents’ taxable income is $65,802 after the standard and non-itemizer charitable deductions."
+us,scenario_123,federal_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the child’s wages in joint AGI and invented $27,300 of mortgage interest from the outstanding balance and an assumed rate. Mortgage principal alone creates no interest deduction, and the parents’ joint AGI is $100,002."
+us,scenario_123,federal_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model stacked the standard deduction with itemized deductions, invented mortgage interest, treated W-2 wages as qualified business income, and fabricated a dependent-care credit without care expenses. It also improperly used refundable-credit concepts to force the before-refundable-credits liability to zero."
+us,scenario_123,federal_refundable_credits,claude-haiku-4.5,llm_error,credit_phaseout,False,"The model treated $1,600 of CTC as automatically refundable instead of first applying the CTC against the household’s federal income tax liability, which leaves no unused amount for the ACTC. It also invented a $205 EITC despite the household’s income exceeding the applicable one-child joint-filer phaseout range."
+us,scenario_123,federal_refundable_credits,gpt-5.4-mini,llm_error,credit_phaseout,False,"The model used the ACTC refundability cap as an automatic payment and asserted there was no limitation from the nonrefundable portion. The household’s tax liability absorbs the available CTC as a nonrefundable credit, so the unused CTC—and therefore the refundable ACTC—is $0."
+us,scenario_123,federal_refundable_credits,inkling,llm_error,credit_phaseout,False,"The model inferred that sufficient earned income guarantees the maximum ACTC, applying only the earned-income phase-in. It omitted the separate limit tying ACTC to unused CTC after the nonrefundable credit offsets tax liability, which yields $0 here."
+us,scenario_123,federal_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,"The model supplied neither the required numeric output nor an explanation for federal_refundable_credits, so its response failed the submission contract."
+us,scenario_123,federal_refundable_credits,ox-alpha,llm_error,credit_phaseout,False,"The model correctly eliminated EITC but treated being below the $400,000 CTC phaseout and above the ACTC earnings floor as sufficient for a $1,700 refund. It failed to subtract the CTC used nonrefundably against federal tax liability; no unused CTC remains refundable."
+us,scenario_123,federal_refundable_credits,qwen3.8-max,llm_error,household_unit_or_filing_status,False,"The model incorrectly used the dependent child’s $45,000 of wages to generate an ACTC for the parents and treated the full CTC cap as refundable. The child’s earnings belong to the child’s tax calculation, while the parents’ available CTC is fully consumed as a nonrefundable credit, leaving refundable credits of $0."
 us,scenario_123,free_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,head_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,head_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model returned no parseable head_medicaid_eligible value or explanation, violating the required function-output contract."
@@ -7269,62 +7630,66 @@ us,scenario_123,head_medicare_eligible,glm-5.2,parse_contract_failure,missing_ou
 us,scenario_123,head_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,local_income_tax,claude-haiku-4.5,llm_error,state_local_rule,False,"The model inferred Philadelphia residence solely from the household's Pennsylvania state and applied Philadelphia's wage tax despite the instruction to treat unlisted locality facts as false. Its unexplained reduction from the computed $5,613.24 to $3,545 also applies no identified exemption or valid computation step."
 us,scenario_123,local_income_tax,claude-sonnet-5,llm_error,state_local_rule,False,"The model treated Philadelphia as the default Pennsylvania locality and taxed both workers' wages even though Philadelphia residence or employment was never listed. It also submitted $3,830.90 after its own calculations produced $5,437.50 or $5,495.50, so the final number does not follow from its stated rate or tax base."
-us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"The model explicitly calculated all three components and their correct $11,194 sum, then submitted $11,663.35 instead. Its error is a final-answer transcription failure disconnected from its own computation."
-us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model correctly found $11,092.50 of federal FICA but omitted Pennsylvania's $101.50 employee unemployment-compensation contribution. It then invented $648.89 of Additional Medicare Tax despite stating that neither the individual nor joint threshold was exceeded."
-us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model calculated Pennsylvania's 0.07% employee unemployment-compensation contribution as $101.50, then replaced that component with zero in the final sum. Adding it to the $11,092.50 federal FICA total yields $11,194."
-us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly computed $7,720 for the head and $3,474 for the child, explicitly summed them to $11,194, and then submitted $11,097.50. The submitted value is a transcription or arithmetic-output error unsupported by its derivation."
-us,scenario_123,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model stopped after computing $11,092.50 of employee Social Security and Medicare tax. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $101.50 on the workers' $145,000 of wages."
-us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,thresholds_rates,False,"The model incorrectly capped Pennsylvania's employee unemployment-compensation contribution at $10,000 of wages per worker, producing only $14. The 0.07% employee contribution applies to the full $145,000 of combined wages here, producing $101.50 and a total payroll tax of $11,194."
-us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no mandatory employee unemployment-insurance withholding. It therefore omitted the state's 0.07% employee contribution of $101.50 from the otherwise correct $11,092.50 federal FICA calculation."
-us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model calculated only employee Social Security and Medicare taxes. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of $101.50, which raises the total from $11,092.50 to $11,194."
-us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,payroll_tax_base,False,"Applying the stated 7.65% FICA rate to $145,000 yields $11,092.50, not $11,108. The model both miscomputed its stated FICA calculation and failed to add the actual $101.50 Pennsylvania employee unemployment-compensation contribution."
-us,scenario_123,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly treated Pennsylvania mandatory employee-side payroll tax as zero. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $101.50 to its correctly calculated $11,092.50 of federal FICA."
-us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,payroll_tax_base,False,"The stated Social Security and Medicare calculation on $145,000 of wages yields $11,092.50, not $12,366. The model also assumed away Pennsylvania's $101.50 employee unemployment-compensation contribution, so its submitted number follows neither the federal arithmetic nor the complete payroll-tax definition."
-us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,payroll_tax_base,False,"Employee Social Security and Medicare tax on $145,000 equals $11,092.50, not $15,399. The model misapplied or miscomputed the federal rates and also omitted Pennsylvania's $101.50 employee unemployment-compensation contribution."
-us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model correctly computed $11,092.50 of federal FICA but stopped there. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $101.50."
-us,scenario_123,payroll_tax,grok-4.3,llm_error,payroll_tax_base,False,"FICA at 7.65% on the two workers' $145,000 of wages is $11,092.50, not $11,107.50. The model also omitted the separately required Pennsylvania employee unemployment-compensation contribution of $101.50."
-us,scenario_123,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model correctly calculated $11,092.50 of employee Social Security and Medicare tax but incorrectly set mandatory Pennsylvania employee payroll tax to zero. The 0.07% unemployment-compensation contribution adds $101.50."
-us,scenario_123,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,The model incorrectly concluded that Pennsylvania imposed no mandatory employee state payroll tax. It omitted the 0.07% employee unemployment-compensation contribution of $101.50 from its otherwise correct federal FICA total.
-us,scenario_123,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model correctly derived $11,092.50 of federal Social Security and Medicare tax but set state mandatory employee payroll taxes to zero. Pennsylvania's employee unemployment-compensation contribution is $101.50 on the combined wages."
-us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax value or explanation. It therefore failed the required structured-output contract rather than completing a substantive calculation.
-us,scenario_123,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model acknowledged that the listed wages generate employee Social Security and Medicare liabilities but discarded that calculation and submitted zero. The wage inputs produce $11,092.50 of federal FICA plus $101.50 of Pennsylvania employee unemployment-compensation tax."
-us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no employee-side state payroll tax. Its federal calculation is $11,092.50, and the omitted 0.07% employee unemployment-compensation contribution adds $101.50."
-us,scenario_123,payroll_tax,qwen3.8-max,llm_error,thresholds_rates,False,"The model mislabeled the head's total FICA as Social Security, counted Medicare a second time, and imposed $270 of Additional Medicare Tax even though the stated wages do not exceed the applicable threshold. It then applied an unsupported rounding adjustment instead of summing $8,990 of Social Security, $2,102.50 of Medicare, and $101.50 of Pennsylvania employee unemployment-compensation tax."
+us,scenario_123,payroll_tax,claude-fable-5,llm_error,other,False,"The model correctly derived every component and explicitly summed them to $11,194, but submitted the unrelated value $11,663.35. This is a final-answer transcription or arithmetic-consistency failure."
+us,scenario_123,payroll_tax,claude-haiku-4.5,llm_error,payroll_tax_base,False,"The model omitted Pennsylvania's $101.50 employee unemployment-compensation contribution and then invented $648.89 of Additional Medicare Tax despite correctly stating that no wages exceed the applicable threshold. Federal FICA is $11,092.50, and adding the Pennsylvania contribution yields $11,194."
+us,scenario_123,payroll_tax,claude-opus-4.7,llm_error,payroll_tax_base,False,"The model calculated Pennsylvania's employee unemployment-compensation contribution as $101.50 but replaced it with zero in the final sum. Adding that mandatory employee state payroll tax to $11,092.50 of federal FICA yields $11,194."
+us,scenario_123,payroll_tax,claude-opus-4.8,llm_error,other,False,"The model correctly calculated federal FICA and Pennsylvania employee unemployment compensation and explicitly obtained $11,194, but submitted $11,097.50. The submitted number is an unsupported final-answer transcription error."
+us,scenario_123,payroll_tax,claude-opus-5,llm_error,payroll_tax_base,False,"The model stopped after calculating $11,092.50 of Social Security and Medicare tax. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution of $101.50."
+us,scenario_123,payroll_tax,claude-sonnet-4.6,llm_error,payroll_tax_base,False,"The model incorrectly capped Pennsylvania's 0.07% employee unemployment-compensation contribution at $10,000 of wages per worker, producing only $14. The contribution applies to the combined $145,000 of covered wages, producing $101.50 and total payroll tax of $11,194."
+us,scenario_123,payroll_tax,claude-sonnet-5,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no employee unemployment-insurance withholding. Pennsylvania's 0.07% employee unemployment-compensation contribution adds $101.50 to the correctly calculated $11,092.50 of federal FICA."
+us,scenario_123,payroll_tax,deepseek-v4-pro,llm_error,payroll_tax_base,False,"The model calculated only Social Security and Medicare taxes. It omitted Pennsylvania's mandatory employee unemployment-compensation contribution of 0.07% of $145,000, or $101.50."
+us,scenario_123,payroll_tax,gemini-3.1-flash-lite-preview,llm_error,other,False,"The model claimed to apply the 7.65% combined FICA rate to $145,000, but that calculation equals $11,092.50 rather than $11,108. It also omitted the $101.50 Pennsylvania employee unemployment-compensation contribution, so the complete total is $11,194."
+us,scenario_123,payroll_tax,glm-5.2,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania has no mandatory employee-side state payroll tax. The state's 0.07% employee unemployment-compensation contribution adds $101.50 to federal FICA of $11,092.50."
+us,scenario_123,payroll_tax,gpt-5.4-mini,llm_error,other,False,"The model described the correct federal tax bases and rates but submitted $12,366, which does not follow from its stated computation. Social Security and Medicare total $11,092.50, and Pennsylvania employee unemployment compensation adds $101.50, yielding $11,194."
+us,scenario_123,payroll_tax,gpt-5.4-nano,llm_error,other,False,"The model's $15,399 does not equal 7.65% of the stated $145,000 wage base and therefore miscomputes federal FICA. Correct FICA is $11,092.50, with another $101.50 due for Pennsylvania employee unemployment compensation."
+us,scenario_123,payroll_tax,gpt-5.6-terra,llm_error,payroll_tax_base,False,"The model stopped at federal FICA of $11,092.50. It omitted Pennsylvania's mandatory 0.07% employee unemployment-compensation contribution, which equals $101.50."
+us,scenario_123,payroll_tax,grok-4.3,llm_error,other,False,"The model miscalculated 7.65% of $145,000 as $11,107.50 instead of $11,092.50. It also omitted Pennsylvania's $101.50 employee unemployment-compensation contribution, producing a complete payroll-tax total of $11,194."
+us,scenario_123,payroll_tax,grok-4.5,llm_error,payroll_tax_base,False,"The model incorrectly treated Pennsylvania mandatory employee state payroll tax as zero. The 0.07% employee unemployment-compensation contribution on $145,000 is $101.50, which must be added to $11,092.50 of federal FICA."
+us,scenario_123,payroll_tax,grok-4.6,llm_error,payroll_tax_base,False,"The model incorrectly asserted that Pennsylvania has no mandatory employee-side state payroll tax. Pennsylvania's employee unemployment-compensation contribution is $101.50, raising the $11,092.50 federal FICA amount to $11,194."
+us,scenario_123,payroll_tax,grok-build-0.1,llm_error,payroll_tax_base,False,"The model omitted Pennsylvania's mandatory employee unemployment-compensation contribution after correctly calculating federal FICA. Applying 0.07% to $145,000 adds $101.50."
+us,scenario_123,payroll_tax,inkling,llm_error,payroll_tax_base,False,"The model incorrectly set mandatory state employee payroll taxes to zero. Pennsylvania's 0.07% employee unemployment-compensation contribution on the two workers' $145,000 of wages equals $101.50."
+us,scenario_123,payroll_tax,kimi-k2.6,parse_contract_failure,missing_output,False,The model supplied no payroll-tax output or explanation. It therefore failed the required structured-output contract rather than completing the calculation.
+us,scenario_123,payroll_tax,minimax-m3,llm_error,payroll_tax_base,False,"The model acknowledged that the listed wages generate employee Social Security and Medicare taxes but then replaced the calculation with an unsupported zero. The wages produce $11,092.50 of federal FICA, and Pennsylvania employee unemployment compensation adds $101.50."
+us,scenario_123,payroll_tax,ox-alpha,llm_error,payroll_tax_base,False,"The model incorrectly stated that Pennsylvania levies no mandatory employee payroll tax. Its employee unemployment-compensation contribution adds $101.50 to the correctly computed $11,092.50 of Social Security and Medicare tax."
+us,scenario_123,payroll_tax,qwen-3.7-max,llm_error,payroll_tax_base,False,"The model incorrectly treated Pennsylvania employee-side payroll tax as zero. The state's 0.07% employee unemployment-compensation contribution on $145,000 adds $101.50 to federal FICA."
+us,scenario_123,payroll_tax,qwen3.8-max,llm_error,other,False,"The model misidentified the head's $6,200 Social Security tax as $7,650, added a $270 Additional Medicare Tax even though the applicable wage threshold is not exceeded, and then applied an unsupported rounding adjustment. The correct components are $8,990 of Social Security, $2,102.50 of Medicare, zero Additional Medicare Tax, and $101.50 of Pennsylvania employee unemployment compensation."
 us,scenario_123,reduced_price_school_meals_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,spouse_chip_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,spouse_medicaid_eligible,glm-5.2,parse_contract_failure,missing_output,False,"The model supplied no spouse_medicaid_eligible output, violating the required submission contract and omitting the engine-derived result of 0."
 us,scenario_123,spouse_medicaid_eligible,qwen3.8-max,llm_error,thresholds_rates,False,"The model incorrectly stated that the spouse's MAGI was below Pennsylvania's adult Medicaid threshold. The spouse's MAGI is 5.31 times FPL, far above the applicable income limits, and the spouse qualifies through no other Medicaid category."
 us,scenario_123,spouse_medicare_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,spouse_wic_eligible,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
-us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model added the child’s separately assessed $45,000 of wages to the parents’ filing-unit income. It also submitted $4,453.06 despite computing $4,451.56, but the controlling error was including the child rather than taxing $100,002 at 3.07%."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model invented Pennsylvania standard deductions of $15,000 and $8,500 even though Pennsylvania provides no standard deduction, and it also combined the child’s wages with the parents’ income. The benchmark base is the parents’ $100,002 without those deductions, producing $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model incorrectly included the child’s separately taxed $45,000 of wages in the benchmark filing unit. It then submitted $4,452.08 instead of its own $4,451.56 calculation; the required computation is $100,002 × 3.07% = $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model treated everyone living in the household as one Pennsylvania income-tax filing unit and included the child’s $45,000. It also submitted $4,452.61 rather than its stated $4,451.56 calculation; only the parents’ $100,002 belongs in this output."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model added the child’s separate $1,381.50 Pennsylvania liability to the parents’ liability. This benchmark output covers the parents’ filing unit, whose $100,002 taxable income yields $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model correctly recognized that the child files separately, then overrode that rule because the people were described as one household group. Cohabitation and benefit-household grouping do not merge the child into the parents’ Pennsylvania filing unit, so the child’s $45,000 must be excluded."
-us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,thresholds_rates,False,"The model derived the exact taxable base and exact tax, $100,002 × 3.07% = $3,070.06, then replaced it with an unsupported approximate $3,060. Pennsylvania’s treatment supplies no adjustment that reduces the correctly computed amount by $10.06."
-us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model added tax on the child’s $45,000 of wages to the parents’ tax and also omitted the parents’ $2 of taxable interest from its detailed calculation. The benchmark filing unit contains $100,002 of taxable income, not the child’s separately assessed earnings."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model equated total income across household residents with Pennsylvania taxable income for the benchmark filing unit. The child’s $45,000 is separately assessed and excluded, leaving $100,002 taxed at 3.07%."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 3% rate instead of Pennsylvania’s 3.07% rate and implicitly taxed all $145,000 of wages, including the child’s separate earnings. The required calculation applies 3.07% to the parents’ $100,002 taxable income."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model incorrectly defined $145,002 as the benchmark filing unit’s taxable income by including the child’s wages. The parents’ Pennsylvania base is $100,002, yielding $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model separately calculated the parents’ correct tax and the child’s tax, then incorrectly summed both filing units into this output. The requested benchmark value is the parents’ $3,070.06 alone."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied Pennsylvania tax to combined resident income of $145,002 rather than the parents’ filing-unit income. Excluding the child’s separately assessed $45,000 leaves the $100,002 base."
-us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 in a single household taxable-income total. The child is outside the parents’ Pennsylvania filing unit for this output, so tax is 3.07% of $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model correctly computed $3,070.06 for the parents but then added $1,381.50 from the child’s separate return. The benchmark output does not aggregate that separate child filing unit."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,household_unit_or_filing_status,False,"The model said it taxed household wages and interest, thereby including the child’s separately assessed earnings, and its $4,807 figure does not follow from 3.07% of the stated income. The benchmark computation is 3.07% of the parents’ $100,002, or $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model imported federal-style itemized-deduction reasoning into Pennsylvania tax and produced $9,055 without a coherent Pennsylvania taxable-income calculation. Pennsylvania does not use the listed medical expenses or real-estate taxes this way; the applicable base is $100,002 and the tax is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 of wages in the parents’ Pennsylvania taxable-income base. Removing that separately assessed income leaves $100,002 subject to the 3.07% rate."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model combined the child’s wages with the parents’ income and also described $145,002 as wages plus an additional $2 of interest, double-counting the interest in its wording. The correct filing-unit base is the parents’ $100,002, and even its stated combined base would not produce the submitted $4,455.06."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model treated the child’s separately assessed $45,000 as part of the parents’ Pennsylvania taxable income. This output applies the 3.07% rate only to the parents’ $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model used all household residents’ wages and interest as one Pennsylvania tax base. The child’s wages belong to a separate filing unit, leaving $100,002 and tax of $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model taxed approximately $145,000 of combined household income, which includes the child’s separate earnings. It needed to apply 3.07% to the parents’ $100,002 without federal-style deductions."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 in combined taxable compensation and interest. The benchmark filing-unit base is $100,002, so the tax before refundable credits is $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model explicitly combined the child’s $45,000 with the parents’ income and rounded the resulting separate-unit aggregate. The child’s earnings are excluded from this filing-unit output, which uses $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model explicitly aggregated tax across two filing units. The benchmark output contains the parents’ filing unit only, so the child’s $45,000 and its $1,381.50 tax are not added."
-us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The submitted $2,939.86 corresponds to applying 3.07% to about $95,761, an unexplained $4,241 reduction from the parents’ $100,002 taxable base. Pennsylvania supplies no listed deduction producing that reduction, so the calculation must use the full $100,002."
-us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model correctly described Pennsylvania’s rate and deduction rules but wrongly included the child’s separately assessed wages in its $145,002 base. The parents’ filing unit has $100,002 of taxable income."
-us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model acknowledged that Pennsylvania tax liability is positive but submitted zero solely because it asserted that the structured output reports zero. Applying 3.07% to the parents’ $100,002 produces $3,070.06, so the submitted value contradicts its own tax analysis."
-us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the child’s $45,000 in the parents’ taxable-income base and also misstated the product of $145,002 and 3.07%, which rounds to $4,451.56 rather than $4,451.57. The benchmark base excludes the child and yields $3,070.06."
-us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model incorrectly substituted a federal-taxable-income concept for Pennsylvania’s class-based taxable income, then submitted $2,565.30 despite stating a calculation of $2,073.02. Pennsylvania taxes the parents’ $100,000 compensation and $2 interest without the federal deductions, producing $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-fable-5,llm_error,household_unit_or_filing_status,False,"The model correctly identified the child's $45,000 as separately filed but then added the child's $1,381.50 liability to the parents' benchmark output. It also submitted $4,453.06 despite computing $4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-haiku-4.5,llm_error,taxable_income_or_deductions,False,"The model wrongly imported federal-style standard deductions into Pennsylvania tax and also included the separately filing child's wages in the household output. Pennsylvania applies 3.07% to the parents' $100,002 without those deductions, yielding $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.7,llm_error,household_unit_or_filing_status,False,"The model included the child's separately taxed $45,000 of wages in the parents' Pennsylvania tax output, inflating the base from $100,002 to $145,002. It then submitted $4,452.08 even though its stated multiplication produced $4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-4.8,llm_error,household_unit_or_filing_status,False,"The model added the child's $45,000 of wages to the parents' joint filing unit instead of limiting this output to the parents' $100,002. It also submitted $4,452.61 rather than its own computed $4,451.56."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-opus-5,llm_error,household_unit_or_filing_status,False,"The model added the child's separate $1,381.50 Pennsylvania liability to the parents' $3,070.06 liability. The benchmark output includes only the parents' joint tax unit."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-4.6,llm_error,household_unit_or_filing_status,False,"The model explicitly derived the parents' correct $3,070.06 liability and recognized that the child files separately, but then overrode that filing-unit treatment because everyone lives in one household. Household membership does not merge the child's separate Pennsylvania return into the parents' tax-unit output."
+us,scenario_123,state_income_tax_before_refundable_credits,claude-sonnet-5,llm_error,other,False,"The model correctly derived $100,002 of Pennsylvania taxable income and $3,070.06 of tax, then replaced that exact result with an unsupported $3,060 adjustment for nonexistent “compensation-only interest treatment nuances.” Both dollars of taxable interest remain in the base."
+us,scenario_123,state_income_tax_before_refundable_credits,deepseek-v4-pro,llm_error,household_unit_or_filing_status,False,"The model added Pennsylvania tax on the child's separately filed $45,000 of wages to the parents' benchmark output. It also omitted the parents' $2 of taxable interest, which contributes $0.06 to the correct $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3-flash-preview,llm_error,household_unit_or_filing_status,False,"The model treated all residents' income as one Pennsylvania taxable-income base and included the child's $45,000. This output uses the parents' joint filing unit and taxes only their $100,002 at 3.07%."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-flash-lite-preview,llm_error,thresholds_rates,False,"The model used a 3% rate instead of Pennsylvania's 3.07% rate and applied it to all $145,000 of household wages, including the separately filing child's wages. The correct computation is 3.07% of the parents' $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.1-pro-preview,llm_error,household_unit_or_filing_status,False,"The model incorrectly defined $145,002 across all household members as the taxable income for this output. The child's $45,000 belongs to a separate filing unit, leaving $100,002 in the parents' joint unit."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.5-flash,llm_error,household_unit_or_filing_status,False,"The model correctly separated the parents' $3,070.06 tax from the child's tax but then summed both filing units into the requested output. Only the parents' joint Pennsylvania liability is included."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.6-flash,llm_error,household_unit_or_filing_status,False,"The model applied 3.07% to income pooled across the entire household, including the child's $45,000. The benchmark output is based on the parents' $100,002 joint taxable income."
+us,scenario_123,state_income_tax_before_refundable_credits,gemini-3.7-flash,llm_error,household_unit_or_filing_status,False,"The model included the child's separately filed wages in a $145,002 household tax base. Excluding that separate filing unit leaves the parents' $100,002 base and $3,070.06 tax."
+us,scenario_123,state_income_tax_before_refundable_credits,glm-5.2,llm_error,household_unit_or_filing_status,False,"The model correctly computed $3,070.06 for the parents and $1,381.50 for the child's separate return, but incorrectly summed them. The requested benchmark output covers the parents' joint tax unit only."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-mini,llm_error,taxable_income_or_deductions,False,"The submitted $4,807 is inconsistent with applying 3.07% to either the parents' $100,002 or the stated household income of $145,002; it implies an unsupported base of about $156,580. The correct base excludes the child's separate wages and equals $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.4-nano,llm_error,taxable_income_or_deductions,False,"The model used an unspecified itemized-deduction calculation and produced $9,055, more than twice the tax generated even by taxing all listed wages at 3.07%. Pennsylvania does not use the listed federal-style medical and property-tax deductions here, and the applicable parents' base is $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.5,llm_error,household_unit_or_filing_status,False,"The model pooled the child's $45,000 of earnings with the parents' wages and interest. The child's earnings are assigned to a separate filing unit and do not enter this $100,002 Pennsylvania tax base."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-luna,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 in the taxable base instead of restricting the output to the parents' joint unit. Its submitted $4,455.06 is also inconsistent with 3.07% of the $145,002 base stated in its explanation."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-sol,llm_error,household_unit_or_filing_status,False,"The model treated $145,002 across the household as one Pennsylvania filing-unit base. Removing the separately filing child's $45,000 leaves the parents' $100,002 taxable income."
+us,scenario_123,state_income_tax_before_refundable_credits,gpt-5.6-terra,llm_error,household_unit_or_filing_status,False,"The model incorrectly pooled the child's earnings into the parents' Pennsylvania income-tax output. The parents' joint filing unit contains $100,002, not $145,002."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.3,llm_error,household_unit_or_filing_status,False,"The model taxed approximately $145,000 of combined household income, thereby including the child's separate earnings. This output applies the 3.07% rate only to the parents' $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.5,llm_error,household_unit_or_filing_status,False,"The model included the child's $45,000 in the parents' taxable-income base and rounded the resulting household-wide calculation. The requested filing-unit liability is exactly $3,070.06 on $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-4.6,llm_error,household_unit_or_filing_status,False,"The model correctly described Pennsylvania's deduction rules but incorrectly combined the child's separate wages with the parents' income. Its rounding to $4,452 occurs only after using that wrong $145,002 base."
+us,scenario_123,state_income_tax_before_refundable_credits,grok-build-0.1,llm_error,household_unit_or_filing_status,False,"The model pooled the head, spouse, and child into a $145,002 Pennsylvania tax base. The child's $45,000 is outside the parents' joint filing unit used for this output."
+us,scenario_123,state_income_tax_before_refundable_credits,inkling,llm_error,household_unit_or_filing_status,False,"The model explicitly summed liabilities across two filing units, including the child's separate $45,000 return. The benchmark state-tax output is the parents' filing-unit liability of $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,kimi-k2.6,llm_error,taxable_income_or_deductions,False,"The $2,939.86 answer corresponds to applying Pennsylvania's 3.07% rate to only about $95,761 rather than the parents' $100,002. The correct derivation permits no adjustment reducing the parents' wages and interest to that implied base."
+us,scenario_123,state_income_tax_before_refundable_credits,kimi-k3,llm_error,household_unit_or_filing_status,False,"The model applied Pennsylvania's rate to all $145,002 earned across household members. The child's $45,000 is taxed in a separate filing unit and is excluded from the requested parents' liability."
+us,scenario_123,state_income_tax_before_refundable_credits,minimax-m3,llm_error,other,False,"The model acknowledged that Pennsylvania tax was positive but submitted zero without performing the calculation. Applying 3.07% to the parents' $100,002 taxable income yields $3,070.06."
+us,scenario_123,state_income_tax_before_refundable_credits,ox-alpha,llm_error,household_unit_or_filing_status,False,"The model correctly rejected the listed deductions but incorrectly included the child's $45,000 in the taxable base. The parents' joint Pennsylvania base is $100,002."
+us,scenario_123,state_income_tax_before_refundable_credits,qwen-3.7-max,llm_error,household_unit_or_filing_status,False,"The model included the child's separately filed wages in a $145,002 base instead of using the parents' $100,002. It also misstated the arithmetic: $145,002 multiplied by 3.07% is $4,451.56 when rounded to cents, not $4,451.57."
+us,scenario_123,state_income_tax_before_refundable_credits,qwen3.8-max,llm_error,taxable_income_or_deductions,False,"The model wrongly substituted federal taxable income for Pennsylvania taxable income even though Pennsylvania does not start from that federally deducted amount. It also submitted $2,565.30 despite stating that its own calculation produced $2,073.02; the required computation is 3.07% of $100,002."
 us,scenario_123,state_refundable_credits,kimi-k2.6,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
 us,scenario_123,tanf,glm-5.2,parse_contract_failure,missing_output,False,All wrong responses were missing or unparseable predictions.
diff --git a/app/src/components/Hero.tsx b/app/src/components/Hero.tsx
index e483365..4f7a31b 100644
--- a/app/src/components/Hero.tsx
+++ b/app/src/components/Hero.tsx
@@ -2,7 +2,7 @@ import { DEFAULT_VERSION_ID } from "../lib/dataVersionsRuntime";
 import type { BenchData, CountryCode } from "../types";
 import SiteHeader, { type HeaderNavItem } from "./SiteHeader";
 
-const SNAPSHOT_DATE_LABEL = "Snapshot 2026-08-17";
+const SNAPSHOT_DATE_LABEL = "Snapshot 2026-08-22";
 
 export default function Hero({
   selectedView,
diff --git a/app/src/data.artifact.json b/app/src/data.artifact.json
index edb1060..2fd0094 100644
--- a/app/src/data.artifact.json
+++ b/app/src/data.artifact.json
@@ -1,9 +1,9 @@
 {
   "version": 1,
   "repo": "PolicyEngine/policybench",
-  "tag": "dashboard-data-20260817",
+  "tag": "dashboard-data-20260822",
   "asset": "dashboard-data.json",
-  "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260817/dashboard-data.json",
-  "sha256": "a71de04ab9b3aa6d20e99fb8c7f90ec60ce4fdb708e2546bfb68ddbd968409fe",
-  "bytes": 80150275
+  "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260822/dashboard-data.json",
+  "sha256": "b883ec669d510ea29c9c18f18c30030c5bbd29f770bcd90d257779940929a895",
+  "bytes": 85042876
 }
\ No newline at end of file
diff --git a/app/src/data.versions.json b/app/src/data.versions.json
index e83581a..77e1e15 100644
--- a/app/src/data.versions.json
+++ b/app/src/data.versions.json
@@ -5,7 +5,7 @@
     {
       "id": "1.1",
       "label": "1.1",
-      "description": "Corrected ground truth (policyengine-us 1.755.4); identical forced-tool request for every model - 30 models",
+      "description": "Corrected ground truth (policyengine-us 1.755.4); identical forced-tool request for every model - 32 models",
       "snapshotLabel": null,
       "artifact": {
         "pointer": "live"
diff --git a/app/src/modelMeta.ts b/app/src/modelMeta.ts
index f6399f8..e22d2f3 100644
--- a/app/src/modelMeta.ts
+++ b/app/src/modelMeta.ts
@@ -13,6 +13,8 @@ export const MODEL_ORDER = [
   "claude-haiku-4.5",
   "grok-4.3",
   "grok-4.5",
+  "grok-4.6",
+  "ox-alpha",
   "grok-build-0.1",
   "gpt-5.6-sol",
   "gpt-5.6-terra",
@@ -47,6 +49,8 @@ export const MODEL_LABELS: Record<string, string> = {
   "claude-sonnet-4.6": "Claude Sonnet 4.6",
   "grok-4.3": "Grok 4.3",
   "grok-4.5": "Grok 4.5",
+  "grok-4.6": "Grok 4.6",
+  "ox-alpha": "Ox Alpha (preview)",
   "grok-build-0.1": "Grok Build 0.1",
   "gpt-5.6-sol": "GPT-5.6 Sol",
   "gpt-5.6-terra": "GPT-5.6 Terra",
@@ -108,6 +112,8 @@ export const MODEL_RELEASE_DATES: Record<string, string> = {
   "gpt-5.6-luna": "2026-07-09",
   "grok-4.3": "2026-04-17",
   "grok-4.5": "2026-07-08",
+  "grok-4.6": "2026-08-12",
+  "ox-alpha": "2026-08-21",
   "grok-build-0.1": "2026-05-29",
   "deepseek-v4-pro": "2026-04-24",
   "deepseek-v4-flash": "2026-04-24",
diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json
index 4a363a8..18ad877 100644
--- a/paper/snapshot/20260501/manifest.json
+++ b/paper/snapshot/20260501/manifest.json
@@ -1,7 +1,7 @@
 {
   "audit_annotation_artifacts": {
     "files": {
-      "us_audit_row_annotations.csv": "9225b168d63ab233a374823587d128efd810ff66b17fe9984538aade2f09ee40",
+      "us_audit_row_annotations.csv": "f4b89cdb075d1725e4f1808a73e36c3eb79c72bc851d2553083f6688da2f5b7c",
       "us_case_notes.csv": "7e340f1dc6e22e2bbae18fd92579eab1d886918725377d3bf28be63f08ef5030",
       "us_case_reference_explanations.csv": "a3fc7504dcf3d8abe11e98c2228c786a5171a825640b980835e1e210d3afa198"
     },
@@ -22,11 +22,11 @@
   ],
   "live_dashboard_artifact": {
     "asset": "dashboard-data.json",
-    "bytes": 80150275,
+    "bytes": 85042876,
     "derivation": "At freeze time the live artifact equals the frozen published_dashboard_artifact: the combined export of the source-run data.json listed under source_run_artifacts. Annotation-class republishes may advance this entry ahead of the frozen pin without changing any score.",
-    "sha256": "a71de04ab9b3aa6d20e99fb8c7f90ec60ce4fdb708e2546bfb68ddbd968409fe",
-    "tag": "dashboard-data-20260817",
-    "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260817/dashboard-data.json"
+    "sha256": "b883ec669d510ea29c9c18f18c30030c5bbd29f770bcd90d257779940929a895",
+    "tag": "dashboard-data-20260822",
+    "url": "https://github.com/PolicyEngine/policybench/releases/download/dashboard-data-20260822/dashboard-data.json"
   },
   "live_dashboard_note": "The live dashboard payload is a published release asset; the committed pointer app/src/data.artifact.json must reference the artifact pinned under live_dashboard_artifact. The separate published_dashboard_artifact freezes the combined export of the source run data.json files listed under source_run_artifacts. A later publication may advance the live entry without changing the frozen pin.",
   "model_response_date": "2026-06-12 to 2026-08-17",
@@ -135,4 +135,4 @@
   "source_run_labels": {
     "us": "us_full_run_20260612_policyengine_4_16_1_populace"
   }
-}
+}
\ No newline at end of file
diff --git a/policybench/config.py b/policybench/config.py
index d5505e5..ced8db1 100644
--- a/policybench/config.py
+++ b/policybench/config.py
@@ -43,6 +43,7 @@
     "claude-haiku-4.5": "claude-haiku-4-5-20251001",
     "grok-4.3": "xai/grok-4.3",
     "grok-4.5": "xai/grok-4.5",
+    "grok-4.6": "xai/grok-4.6",
     "grok-build-0.1": "xai/grok-build-0.1",
     **GPT_56_MODELS,
     "gpt-5.5": "gpt-5.5",
@@ -52,6 +53,9 @@
     "gemini-3.5-flash": "gemini/gemini-3.5-flash",
     "gemini-3.6-flash": "gemini/gemini-3.6-flash",
     "gemini-3.7-flash": "gemini/gemini-3.7-flash",
+    # Cloaked OpenRouter preview listed 2026-08-21 (free window); maker
+    # unconfirmed. On the board as a labeled preview row.
+    "ox-alpha": "openrouter/stealth/ox-alpha",
     "gemini-3-flash-preview": "gemini/gemini-3-flash-preview",
     "gemini-3.1-flash-lite-preview": "gemini/gemini-3.1-flash-lite-preview",
     "deepseek-v4-pro": "deepseek/deepseek-v4-pro",
diff --git a/policybench/model_cards.py b/policybench/model_cards.py
index 5d51d73..5f734d1 100644
--- a/policybench/model_cards.py
+++ b/policybench/model_cards.py
@@ -159,6 +159,20 @@ class ModelCard:
             "tail scenarios."
         ),
     ),
+    "xai/grok-4.6": ModelCard(
+        litellm_id="xai/grok-4.6",
+        answer_contract="tool",
+        request_timeout_seconds=600,
+        thinking_budget=True,
+        expected_cost_per_scenario_usd=0.07,
+        notes=(
+            "Onboarded 2026-08-20: forced tool contract passed 3/3 and "
+            "16/16 whole-scenario. Heavy reasoner — the 3-variable probe "
+            "spent 26,260 tokens over 391s, so the timeout is 600s and the "
+            "thinking-class budget applies. Released 2026-08-12 at $2/$6 "
+            "per 1M (litellm map). The 100-scenario run cost $8.70."
+        ),
+    ),
     "deepseek/deepseek-v4-pro": ModelCard(
         litellm_id="deepseek/deepseek-v4-pro",
         answer_contract="json",
@@ -211,6 +225,21 @@ class ModelCard:
             "2027-01-01."
         ),
     ),
+    "openrouter/stealth/ox-alpha": ModelCard(
+        litellm_id="openrouter/stealth/ox-alpha",
+        answer_contract="tool",
+        request_timeout_seconds=600,
+        thinking_budget=True,
+        provider_max_completion_tokens=131_072,
+        expected_cost_per_scenario_usd=0.0,
+        notes=(
+            "Cloaked OpenRouter preview listed 2026-08-21 (free window); "
+            "maker unconfirmed. Board row labeled as a preview: public and "
+            "callable under the identical request, dated by its listing. "
+            "Tool contract passed the 2026-08-20 gauntlet 3/3 and 16/16 "
+            "after the thinking-class budget."
+        ),
+    ),
     "openrouter/moonshotai/kimi-k3": ModelCard(
         litellm_id="openrouter/moonshotai/kimi-k3",
         answer_contract="json",
diff --git a/sensitivity/ox-alpha-2026-08.md b/sensitivity/ox-alpha-2026-08.md
new file mode 100644
index 0000000..31300f6
--- /dev/null
+++ b/sensitivity/ox-alpha-2026-08.md
@@ -0,0 +1,73 @@
+# Ox Alpha preview run (August 2026)
+
+`stealth/ox-alpha` is a cloaked model OpenRouter listed on 2026-08-21 with
+no maker named, a 1M-token context, and a free window of about a week.
+Community fingerprinting points to a Zhipu GLM lineage. Nothing is
+confirmed. We ran it through the full benchmark because free previews of
+unreleased frontier models are worth measuring while they exist.
+
+It sits on the board as a labeled preview row. It is publicly callable
+under the identical request, so it meets the board's inclusion rule; what
+it lacks is a named maker and durability. The row is dated by its
+OpenRouter listing (2026-08-21). If the model later ships under its own
+name, that model gets onboarded normally, with this run as its early read.
+
+## Result
+
+Canonical v1 condition: identical whole-scenario prompt, forced answer
+tool call, no reasoning parameters, 100 households, 18 outputs.
+
+| | exact | would rank | within 1% | parsed |
+|---|---|---|---|---|
+| ox-alpha | **84.2** | #4 of 32 | 86.4 | 1,984 / 1,984 |
+
+Only GPT-5.6 Sol (88.7), Kimi K3 (86.2), and GPT-5.6 Luna (84.5) score
+higher on the board. Inference cost was $0 in the free window; the
+tool contract passed the gauntlet 3/3 and 16/16 after a thinking-class
+completion budget (the 384-token probe budget starved it, the known
+reasoning-model trap).
+
+## Per program
+
+Within-$1 rates per output. GLM-5.2, the latest released model in the
+suspected lineage, appears for reference only.
+
+| program | Ox Alpha | GLM-5.2 |
+|---|---|---|
+| local_income_tax | 100.0 | 94.0 |
+| person_early_head_start_eligible | 100.0 | 89.5 |
+| person_head_start_eligible | 100.0 | 89.5 |
+| person_wic_eligible | 100.0 | 92.7 |
+| person_medicare_eligible | 99.4 | 89.3 |
+| reduced_price_school_meals_eligible | 99.0 | 94.0 |
+| self_employment_tax | 99.0 | 91.0 |
+| tanf | 99.0 | 94.0 |
+| free_school_meals_eligible | 98.0 | 93.0 |
+| ssi | 98.0 | 90.0 |
+| person_medicaid_eligible | 93.8 | 85.9 |
+| person_chip_eligible | 92.7 | 93.2 |
+| payroll_tax | 87.0 | 73.0 |
+| federal_refundable_credits | 84.0 | 79.0 |
+| snap | 79.0 | 76.0 |
+| state_refundable_credits | 79.0 | 72.0 |
+| federal_income_tax_before_refundable_credits | 63.0 | 43.0 |
+| state_income_tax_before_refundable_credits | 55.0 | 45.0 |
+
+On the 20 SNAP cases where the household is owed benefits, Ox Alpha
+answers exactly $0 seven times and gets none exact — the same failure
+shape as the rest of the board.
+
+## Reproducing
+
+```
+python -m policybench.cli run \
+  --model ox-alpha \
+  --scenario-manifest paper/snapshot/20260501/us_scenarios.csv \
+  --run-dir results/local/oxalpha/run \
+  --budget-usd 5 --max-workers 4
+```
+
+The model card (`openrouter/stealth/ox-alpha`) pins the tool contract, a
+600s timeout, and the thinking-class budget. Predictions are attached to
+the `dashboard-data-20260817` release as
+`preview-ox-alpha-predictions.csv.gz`.
diff --git a/tests/test_model_cards.py b/tests/test_model_cards.py
index bb485be..4ebf79b 100644
--- a/tests/test_model_cards.py
+++ b/tests/test_model_cards.py
@@ -41,6 +41,8 @@
     "gpt-5.4-nano": ("tool", None, 20, 4_096),
     "xai/grok-4.3": ("tool", None, 420, 4_096),
     "xai/grok-4.5": ("tool", None, 420, 16_384),
+    "xai/grok-4.6": ("tool", None, 600, 16_384),
+    "openrouter/stealth/ox-alpha": ("tool", None, 600, 16_384),
     "xai/grok-build-0.1": ("tool", None, 420, 4_096),
     "deepseek/deepseek-v4-pro": ("json", None, 300, 16_384),
     "deepseek/deepseek-v4-flash": ("json", None, 300, 16_384),