Abstract
With their growing capabilities, generative large language models (LLMs) are being increasingly investigated for complex medical tasks. However, their effectiveness in real-world clinical applications remains underexplored. To address this, we present CliniBench, the first benchmark that enables comparability of well-studied encoder-based classifiers and generative LLMs for discharge diagnosis prediction from admission notes in MIMIC-IV dataset. Our extensive study compares 12 generative LLMs and 3 encoder-based classifiers and demonstrates that encoder-based classifiers consistently outperform generative models in diagnosis prediction. We assess several retrieval augmentation strategies for in-context learning from similar patients and find that they provide notable performance improvements for generative LLMs.
BibTeX
@inproceedings{grundmann-etal-2026-clinibench,
title={CliniBench: A Clinical Outcome Prediction Benchmark for Generative and Encoder-Based Language Models},
author={Paul Grundmann and Jan Frick and Dennis Fast and Thomas Steffek and Felix Gers and Wolfgang Nejdl and Alexander Löser},
editor={Vera Demberg and Kentaro Inui and Lluís Marquez},
booktitle={Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 1: Long Papers)},
month={March},
year={2026},
address={Rabat, Morocco},
publisher={Association for Computational Linguistics},
url={https://aclanthology.org/2026.eacl-long.247/},
doi={10.18653/v1/2026.eacl-long.247},
pages={5360--5378},
ISBN={979-8-89176-380-7},
abstract={With their growing capabilities, generative large language models (LLMs) are being increasingly investigated for complex medical tasks. However, their effectiveness in real-world clinical applications remains underexplored. To address this, we present CliniBench, the first benchmark that enables comparability of well-studied encoder-based classifiers and generative LLMs for discharge diagnosis prediction from admission notes in the MIMIC-IV dataset. Our extensive study compares 12 generative LLMs and 3 encoder-based classifiers and demonstrates that encoder-based classifiers consistently outperform generative models in diagnosis prediction. We assess several retrieval augmentation strategies for in-context learning from similar patients and find that they provide notable performance improvements for generative LLMs.},
}