@mastersthesis{design-gsi-mastersthesis-20264,
author = "L{\'o}pez Escobar, Carlota ",
abstract = "Large language models (LLMs) have demonstrated a remarkable ability to solve natural language processing (NLP) tasks. However, when faced with complex reasoning tasks, they exhibit limitations associated with their probabilistic nature, such as hallucinations or the propagation of chain errors throughout the reasoning process.
This final degree project proposes to address these limitations through a Multi-Agent System (MAS) based on LLMs for collaborative decision-making. The system pits multiple instances of the same base model against each other in a structured debate, assigning each agent a role with opposing perspectives, with the aim of producing higher-quality decisions than those of a single model.
The system has been implemented as a modular pipeline of four modules (orchestrator, multi-agent simulation engine, baseline, and evaluation) and has been evaluated on 40 dilemmas from the Google BIG-bench benchmark in three categories: causal judgment, moral permissibility, and simple ethical questions. The explored parameter space includes two base models (llama3.2:3b and mistral-small3.2:24b), two temperatures (0.1 and 0.7), three group sizes (2, 3, and 4 agents), and three turn policies (round-robin, random, and moderator), generating 4,800 scenarios in total.
The evaluation combines the LLM-as-a-Judge paradigm with Granite 4.1 as the primary judge, deterministic NLP metrics (BERTScore, BLEU, and ROUGE-L), and conversational dynamics metrics. To validate the reliability of the primary judge, a second independent judge (Command-R, Cohere) was incorporated, and the agreement between them was quan- tified using Cohen’s Kappa index.
The results show that the multi-agent system achieves an overall accuracy of 54.65%, compared to the baseline of 43.12%. This improvement is particularly noticeable in step-by- step reasoning tasks, where debate forces agents to question their initial response. However, analysis by configuration reveals that the improvement is not uniform and depends on the task type, suggesting that structured debate is not always the optimal strategy.s
",
institution = "Universidad Polit{\'e}cnica de Madrid",
title = "{D}esign and {E}valuation of a {M}ulti-{A}gent {S}ystem based on {LLM}s for {C}ollaborative {D}ecision-makings",
year = "2026",
}