AcuityBench: Evaluating Clinical Acuity Identification and Uncertainty Alignment

, ')}. Output only the level.\n\nPresentation: ${c.presentation}`; const response = await model.complete(prompt); return { caseId: c.id, format: 'QA', predictedLevel: this.parseAcuityLevel(response), confidence: 1.0, // QA format implies forced choice }; })); }

private async runChatEvaluation(cases: AcuityCase[], model: LLMInterface): Promise<TriagePrediction[]> { // Conversational mode allows nuance, reducing over-triage but risking under-triage return Promise.all(cases.map(async (c) => { const prompt = A patient presents with: "${c.presentation}". Provide advice on the appropriate level of care.; const response = await model.complete(prompt);

  // Use rubric judge to extract acuity from free text
  const adjudication = await this.rubricJudge.adjudicate(response);
  return {
    caseId: c.id,
    format: 'CONVERSATIONAL',
    predictedLevel: adjudication.level,
    confidence: adjudication.confidence,
    rationale: response,
  };
}));

}

private generateReport(cases: AcuityCase[], predictions: TriagePrediction[], format: 'QA' | 'CONVERSATIONAL'): EvaluationReport { let correct = 0; let overTriage = 0; let underTriage = 0; let calibrationSum = 0; let ambiguousCount = 0;

predictions.forEach((pred) => {
  const caseData = cases.find(c => c.id === pred.caseId);
  if (!caseData?.consensusLabel) return;

  // Accuracy calculation
  if (pred.predictedLevel === caseData.consensusLabel) correct++;

  // Risk analysis: Over-triage is recommending higher care than needed; Under-triage is lower.
  // Order: SELF_CARE < PRIMARY_CARE < URGENT_CARE < EMERGENCY
  const predIdx = Object.values(AcuityLevel).indexOf(pred.predictedLevel);
  const trueIdx = Object.values(AcuityLevel).indexOf(caseData.consensusLabel);

  if (predIdx > trueIdx) overTriage++;
  if (predIdx < trueIdx) underTriage++;

  // Uncertainty calibration on ambiguous cases
  if (caseData.isAmbiguous && caseData.physicianUncertaintyScore !== undefined) {
    // Model confidence is inverse of uncertainty. 
    // High physician uncertainty should correlate with low model confidence.
    const modelUncertainty = 1.0 - pred.confidence;
    calibrationSum += Math.abs(modelUncertainty - caseData.physicianUncertaintyScore);
    ambiguousCount++;
  }
});

const total = predictions.length;
return {
  format,
  accuracy: correct / total,
  overTriageRate: overTriage / total,
  underTriageRate: underTriage / total,
  calibrationError: ambiguousCount > 0 ? calibrationSum / ambiguousCount : 0,
};

}

private parseAcuityLevel(text: string): AcuityLevel { // Robust parsing logic to handle model output variations const match = text.match(/(SELF_CARE|PRIMARY_CARE|URGENT_CARE|EMERGENCY)/i); if (match) return match[1].toUpperCase() as AcuityLevel; return AcuityLevel.PRIMARY_CARE; // Default fallback } }

class RubricJudge { async adjudicate(response: string): Promise<{ level: AcuityLevel; confidence: number }> { // Implementation uses a strict rubric anchored to the 4-level framework // to extract acuity from free-form text without hallucination. // In production, this would invoke a secondary model or rule-based parser. return { level: AcuityLevel.PRIMARY_CARE, confidence: 0.8 }; } }

interface LLMInterface { complete(prompt: string): Promise<string>; }

### Pitfall Guide

1.  **Format-Induced Bias**
    *   *Explanation:* Evaluating only in QA mode hides under-triage risks; evaluating only in chat mode hides over-triage risks. The benchmark shows these formats produce opposite error profiles.
    *   *Fix:* Always run dual-format evaluations. Report metrics separately for QA and conversational modes to understand the full risk landscape.

2.  **The Consensus Trap**
    *   *Explanation:* Focusing solely on the 697 consensus cases gives a false sense of model capability. The 217 ambiguous cases reveal where models fail to align with clinical uncertainty.
    *   *Fix:* Include a dedicated "Ambiguity Stress Test" in your evaluation suite. Measure model behavior specifically on cases with high physician uncertainty scores.

3.  **Over-Confidence in Ambiguity**
    *   *Explanation:* Models tend to produce concentrated predictions even when cases are ambiguous, unlike physicians who express doubt. This leads to dangerous over-confidence.
    *   *Fix:* Implement uncertainty calibration metrics. Penalize models that output high confidence on ambiguous inputs. Consider techniques like temperature scaling or ensemble methods to better reflect uncertainty.

4.  **Under-Triage Blindness**
    *   *Explanation:* Teams often prioritize accuracy or over-triage reduction, ignoring that under-triage is the critical safety failure. A model that sends a cardiac patient home is far worse than one that sends a cold patient to the ER.
    *   *Fix:* Weight evaluation metrics by risk severity. Use cost-sensitive analysis where under-triage errors incur a much higher penalty than over-triage errors.

5.  **Rubric Subjectivity**
    *   *Explanation:* Evaluating free-form responses with vague prompts leads to inconsistent adjudication.
    *   *Fix:* Use a rubric-based judge anchored to the explicit four-level framework. Ensure the rubric definitions are mutually exclusive and collectively exhaustive to minimize adjudication variance.

6.  **Label Drift Across Datasets**
    *   *Explanation:* Different datasets use varying terminology for acuity levels, leading to inconsistent evaluation if not harmonized.
    *   *Fix:* Map all dataset labels to a unified schema before evaluation. The benchmark harmonized five datasets into a shared four-level framework; replicate this process for your data.

7.  **Ignoring Error Direction**
    *   *Explanation:* Reporting only accuracy masks whether the model tends to over-triage or under-triage.
    *   *Fix:* Track error direction explicitly. Report over-triage and under-triage rates separately to understand the model's safety bias.

### Production Bundle

#### Action Checklist

- [ ] **Define Acuity Schema:** Establish a clear four-level acuity framework (e.g., Self-Care, Primary Care, Urgent Care, Emergency) and map all data to this schema.
- [ ] **Collect Ambiguous Cases:** Curate a set of physician-confirmed ambiguous cases to test uncertainty alignment, not just accuracy.
- [ ] **Implement Dual-Format Eval:** Build evaluation pipelines for both structured QA and free-form conversational responses.
- [ ] **Deploy Rubric Judge:** Integrate a rubric-based adjudicator for free-form responses, anchored to the acuity schema.
- [ ] **Measure Calibration:** Calculate uncertainty calibration error by comparing model confidence distributions against physician uncertainty scores.
- [ ] **Analyze Risk Tradeoffs:** Review over-triage and under-triage rates separately for each format to identify safety biases.
- [ ] **Stress Test Edge Cases:** Run additional tests on high-acuity cases and maximally ambiguous cases to probe failure modes.
- [ ] **Document Error Patterns:** Record model-specific error directions and calibration issues to inform deployment decisions.

#### Decision Matrix

| Scenario | Recommended Approach | Why | Cost Impact |
| :--- | :--- | :--- | :--- |
| **High-Risk Triage Interface** | Dual-Format Eval + Uncertainty Calibration | Ensures comprehensive safety assessment; catches under-triage risks in chat and over-triage in QA. | High (More eval complexity) |
| **Low-Risk Health Info Bot** | QA Format Eval | Simpler evaluation; over-triage is less critical; accuracy is primary concern. | Low |
| **Model Selection** | Compare Calibration Error + Under-Triage Rate | Accuracy alone is insufficient; calibration and under-triage are key safety indicators. | Medium |
| **Ambiguous Case Handling** | Route to Human or Conservative Default | Models fail to align with clinical uncertainty; human oversight or conservative routing is safer. | High (Operational cost) |

#### Configuration Template

```yaml
# acuity-eval-config.yaml
schema:
  levels:
    - SELF_CARE
    - PRIMARY_CARE
    - URGENT_CARE
    - EMERGENCY
  hierarchy:
    - SELF_CARE
    - PRIMARY_CARE
    - URGENT_CARE
    - EMERGENCY

evaluation:
  formats:
    - QA
    - CONVERSATIONAL
  metrics:
    - accuracy
    - overTriageRate
    - underTriageRate
    - calibrationError
  weights:
    underTriagePenalty: 5.0
    overTriagePenalty: 1.0

ambiguous_cases:
  threshold: 0.7 # Physician uncertainty score threshold
  count: 217

rubric:
  anchored_levels: true
  adjudication_model: "rubric-judge-v1"

Quick Start Guide

1. Install Dependencies: Set up the evaluation environment with TypeScript and required LLM interfaces.
2. Load Data: Import your harmonized dataset, ensuring cases include isAmbiguous flags and physicianUncertaintyScore where available.
3. Run Evaluation: Execute the AcuityEvaluator on your target models, generating reports for both QA and conversational formats.
4. Review Results: Analyze the EvaluationReport objects, focusing on underTriageRate, overTriageRate, and calibrationError.
5. Iterate: Use insights to refine prompts, adjust model parameters, or implement routing logic for ambiguous cases. Re-evaluate to measure improvement.