{ "schema": "clearbar.verdict.v1", "slug": "banking77-intent", "title": "Banking customer-message intent classification (77 intents)", "task": "classification", "ran_at": "2026-11-04T10:31:02.474Z", "price_snapshot_at": "2026-11-04T10:31:02.471Z", "dataset": { "file": "data/banking77-intent.jsonl", "name": "BANKING77 (PolyAI/banking77), test split", "source": "https://huggingface.co/datasets/PolyAI/banking77", "licence": "CC BY 4.0", "sample": "150 rows drawn with a fixed seed from the 3,080-row test split", "rows": 150 }, "threshold": 0.7, "margin": 0.05, "retries": 2, "temperature": 0, "max_tokens": 256, "system_prompt": "You route banking customer messages. Choose the single intent from the label list that best matches the message. Reply with the label exactly as written and nothing else.", "baseline": "openai/gpt-4.1", "models": [ { "model": "openai/gpt-4.1", "is_baseline": true, "cells": 150, "successes": 115, "pass_rate": 0.7667, "wilson_lo": 0.6928, "wilson_hi": 0.8272, "attempts": 150, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "errors": 0, "cost_total": 0.141974, "cost_per_success": 0.001235, "cost_per_1k_tasks": 0.946493, "cost_estimated": false, "latency_p50_ms": 407, "latency_p95_ms": 625, "status": "borderline", "paired": null, "price": { "prompt_per_mtok": 2, "completion_per_mtok": 8 } }, { "model": "openai/gpt-4.1-mini", "is_baseline": false, "cells": 150, "successes": 113, "pass_rate": 0.7533, "wilson_lo": 0.6786, "wilson_hi": 0.8154, "attempts": 150, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "errors": 0, "cost_total": 0.028382, "cost_per_success": 0.000251, "cost_per_1k_tasks": 0.189213, "cost_estimated": false, "latency_p50_ms": 698, "latency_p95_ms": 1085, "status": "borderline", "paired": { "b": 8, "c": 6, "n": 150, "diff": -0.0133, "p": 0.082837, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.4, "completion_per_mtok": 1.6 } }, { "model": "openai/gpt-4.1-nano", "is_baseline": false, "cells": 150, "successes": 87, "pass_rate": 0.58, "wilson_lo": 0.5, "wilson_hi": 0.656, "attempts": 156, "attempts_per_task": 1.04, "retries_total": 6, "retry_cost_share": 0.0388, "invalid_final": 3, "errors": 0, "cost_total": 0.007376, "cost_per_success": 0.000085, "cost_per_1k_tasks": 0.049175, "cost_estimated": false, "latency_p50_ms": 911, "latency_p95_ms": 1404, "status": "below bar", "paired": { "b": 32, "c": 4, "n": 150, "diff": -0.1867, "p": 0.99997, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.4 } }, { "model": "anthropic/claude-haiku-5.5", "is_baseline": false, "cells": 150, "successes": 121, "pass_rate": 0.8067, "wilson_lo": 0.7361, "wilson_hi": 0.8619, "attempts": 156, "attempts_per_task": 1.04, "retries_total": 6, "retry_cost_share": 0.0606, "invalid_final": 0, "errors": 0, "cost_total": 0.016913, "cost_per_success": 0.00014, "cost_per_1k_tasks": 0.112757, "cost_estimated": false, "latency_p50_ms": 301, "latency_p95_ms": 1339, "status": "cleared", "paired": { "b": 4, "c": 10, "n": 150, "diff": 0.04, "p": 0.000177, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.5 } }, { "model": "google/gemini-2.5-flash-lite", "is_baseline": false, "cells": 150, "successes": 99, "pass_rate": 0.66, "wilson_lo": 0.581, "wilson_hi": 0.731, "attempts": 186, "attempts_per_task": 1.24, "retries_total": 36, "retry_cost_share": 0.0564, "invalid_final": 1, "errors": 12, "cost_total": 0.008415, "cost_per_success": 0.000085, "cost_per_1k_tasks": 0.056103, "cost_estimated": false, "latency_p50_ms": 427, "latency_p95_ms": 604, "status": "below bar", "paired": { "b": 19, "c": 3, "n": 150, "diff": -0.1067, "p": 0.989762, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.4 } }, { "model": "meta-llama/llama-3.3-70b-instruct", "is_baseline": false, "cells": 150, "successes": 116, "pass_rate": 0.7733, "wilson_lo": 0.7, "wilson_hi": 0.833, "attempts": 150, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "errors": 0, "cost_total": 0.009505, "cost_per_success": 0.000082, "cost_per_1k_tasks": 0.063369, "cost_estimated": false, "latency_p50_ms": 458, "latency_p95_ms": 3285, "status": "borderline", "paired": { "b": 6, "c": 7, "n": 150, "diff": 0.0067, "p": 0.009649, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.32 } }, { "model": "mistralai/mistral-small-3.2-24b-instruct", "is_baseline": false, "cells": 150, "successes": 101, "pass_rate": 0.6733, "wilson_lo": 0.5948, "wilson_hi": 0.7432, "attempts": 151, "attempts_per_task": 1.0067, "retries_total": 1, "retry_cost_share": 0.0073, "invalid_final": 0, "errors": 0, "cost_total": 0.007023, "cost_per_success": 0.00007, "cost_per_1k_tasks": 0.046821, "cost_estimated": false, "latency_p50_ms": 631, "latency_p95_ms": 5664, "status": "below bar", "paired": { "b": 17, "c": 3, "n": 150, "diff": -0.0933, "p": 0.972539, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.075, "completion_per_mtok": 0.2 } }, { "model": "deepseek/deepseek-v4-flash", "is_baseline": false, "cells": 150, "successes": 113, "pass_rate": 0.7533, "wilson_lo": 0.6786, "wilson_hi": 0.8154, "attempts": 179, "attempts_per_task": 1.1933, "retries_total": 29, "retry_cost_share": 0.2746, "invalid_final": 8, "errors": 0, "cost_total": 0.011924, "cost_per_success": 0.000106, "cost_per_1k_tasks": 0.079492, "cost_estimated": false, "latency_p50_ms": 333, "latency_p95_ms": 3054, "status": "borderline", "paired": { "b": 6, "c": 4, "n": 150, "diff": -0.0133, "p": 0.027464, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.091, "completion_per_mtok": 0.182 } } ], "verdict": { "kind": "switch", "model": "meta-llama/llama-3.3-70b-instruct", "matches_baseline": true, "cost_per_success": 0.000082, "baseline_cost_per_success": 0.001235, "saving_pct": 0.9336 }, "payback": [ { "monthly_bill": 200, "monthly_saving": 186.72, "days_to_repay": [ { "price": 19, "days": 3.1 }, { "price": 49, "days": 7.9 } ] }, { "monthly_bill": 500, "monthly_saving": 466.8, "days_to_repay": [ { "price": 19, "days": 1.2 }, { "price": 49, "days": 3.1 } ] }, { "monthly_bill": 2000, "monthly_saving": 1867.21, "days_to_repay": [ { "price": 19, "days": 0.3 }, { "price": 49, "days": 0.8 } ] } ], "failures": { "openai/gpt-4.1": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "lost_or_stolen_card", "outcome": "wrong", "reason": "predicted lost_or_stolen_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-28", "input": "Some of my card payments look different than where I purchased products from. Why is that?", "expected": "card_payment_not_recognised", "got": "extra_charge_on_statement", "outcome": "wrong", "reason": "predicted extra_charge_on_statement", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 1 }, { "row": "b77-38", "input": "I would appreciate that someone let me know when there is an extra charge for payments. I was checking the app earlier and saying a charge on one of my payments that was additional that no one made me aware of before.", "expected": "card_payment_fee_charged", "got": "extra_charge_on_statement", "outcome": "wrong", "reason": "predicted extra_charge_on_statement", "attempts": 1 }, { "row": "b77-42", "input": "When will the card arrive?", "expected": "card_delivery_estimate", "got": "card_arrival", "outcome": "wrong", "reason": "predicted card_arrival", "attempts": 1 }, { "row": "b77-47", "input": "how much do you charge to accept transfers", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-49", "input": "Can I also transfer with SWIFT?", "expected": "top_up_by_bank_transfer_charge", "got": "country_support", "outcome": "wrong", "reason": "predicted country_support", "attempts": 1 }, { "row": "b77-53", "input": "Can I give my friends access to my account so they can top it up for me?", "expected": "topping_up_by_card", "got": "beneficiary_not_allowed", "outcome": "wrong", "reason": "predicted beneficiary_not_allowed", "attempts": 1 }, { "row": "b77-54", "input": "Are there restrictions for my disposable card since it does not seem to be working?", "expected": "virtual_card_not_working", "got": "disposable_card_limits", "outcome": "wrong", "reason": "predicted disposable_card_limits", "attempts": 1 } ], "openai/gpt-4.1-mini": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "compromised_card", "outcome": "wrong", "reason": "predicted compromised_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-22", "input": "Is there any place I cannot use my card?", "expected": "card_acceptance", "got": "country_support", "outcome": "wrong", "reason": "predicted country_support", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 1 }, { "row": "b77-38", "input": "I would appreciate that someone let me know when there is an extra charge for payments. I was checking the app earlier and saying a charge on one of my payments that was additional that no one made me aware of before.", "expected": "card_payment_fee_charged", "got": "extra_charge_on_statement", "outcome": "wrong", "reason": "predicted extra_charge_on_statement", "attempts": 1 }, { "row": "b77-42", "input": "When will the card arrive?", "expected": "card_delivery_estimate", "got": "card_arrival", "outcome": "wrong", "reason": "predicted card_arrival", "attempts": 1 }, { "row": "b77-47", "input": "how much do you charge to accept transfers", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-49", "input": "Can I also transfer with SWIFT?", "expected": "top_up_by_bank_transfer_charge", "got": "country_support", "outcome": "wrong", "reason": "predicted country_support", "attempts": 1 }, { "row": "b77-53", "input": "Can I give my friends access to my account so they can top it up for me?", "expected": "topping_up_by_card", "got": "automatic_top_up", "outcome": "wrong", "reason": "predicted automatic_top_up", "attempts": 1 }, { "row": "b77-54", "input": "Are there restrictions for my disposable card since it does not seem to be working?", "expected": "virtual_card_not_working", "got": "disposable_card_limits", "outcome": "wrong", "reason": "predicted disposable_card_limits", "attempts": 1 } ], "openai/gpt-4.1-nano": [ { "row": "b77-0", "input": "How do I pay by check?", "expected": "top_up_by_cash_or_cheque", "got": "balance_not_updated_after_cheque_or_cash_deposit", "outcome": "wrong", "reason": "predicted balance_not_updated_after_cheque_or_cash_deposit", "attempts": 1 }, { "row": "b77-2", "input": "There was an error in exchange rate for my cash withdrawal", "expected": "wrong_exchange_rate_for_cash_withdrawal", "got": "exchange_rate", "outcome": "wrong", "reason": "predicted exchange_rate", "attempts": 1 }, { "row": "b77-3", "input": "I would rather the Mastercard.", "expected": "visa_or_mastercard", "got": "card_about_to_expire", "outcome": "wrong", "reason": "predicted card_about_to_expire", "attempts": 1 }, { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-4", "input": "I'm at an ATM and withdrew 30 pounds and was only given 10. What should I do?", "expected": "wrong_amount_of_cash_received", "got": "cash_withdrawal_not_recognised", "outcome": "wrong", "reason": "predicted cash_withdrawal_not_recognised", "attempts": 1 }, { "row": "b77-6", "input": "Why is the cash deposit not showing up in my account?", "expected": "balance_not_updated_after_cheque_or_cash_deposit", "got": "balance_not_updated_after_bank_transfer", "outcome": "wrong", "reason": "predicted balance_not_updated_after_bank_transfer", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "lost_or_stolen_card", "outcome": "wrong", "reason": "predicted lost_or_stolen_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-16", "input": "In what ATM can I change my pin?", "expected": "change_pin", "got": "atm_support", "outcome": "wrong", "reason": "predicted atm_support", "attempts": 1 }, { "row": "b77-17", "input": "There is more than one of the same transaction on my account.", "expected": "transaction_charged_twice", "got": "failed_transfer", "outcome": "wrong", "reason": "predicted failed_transfer", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-28", "input": "Some of my card payments look different than where I purchased products from. Why is that?", "expected": "card_payment_not_recognised", "got": "card_payment_wrong_exchange_rate", "outcome": "wrong", "reason": "predicted card_payment_wrong_exchange_rate", "attempts": 1 } ], "anthropic/claude-haiku-5.5": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "lost_or_stolen_card", "outcome": "wrong", "reason": "predicted lost_or_stolen_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 1 }, { "row": "b77-42", "input": "When will the card arrive?", "expected": "card_delivery_estimate", "got": "card_arrival", "outcome": "wrong", "reason": "predicted card_arrival", "attempts": 1 }, { "row": "b77-47", "input": "how much do you charge to accept transfers", "expected": "top_up_by_bank_transfer_charge", "got": "receiving_money", "outcome": "wrong", "reason": "predicted receiving_money", "attempts": 1 }, { "row": "b77-49", "input": "Can I also transfer with SWIFT?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_into_account", "outcome": "wrong", "reason": "predicted transfer_into_account", "attempts": 1 }, { "row": "b77-51", "input": "I believe someone is using my card without my agreement!", "expected": "compromised_card", "got": "card_payment_not_recognised", "outcome": "wrong", "reason": "predicted card_payment_not_recognised", "attempts": 1 }, { "row": "b77-54", "input": "Are there restrictions for my disposable card since it does not seem to be working?", "expected": "virtual_card_not_working", "got": "disposable_card_limits", "outcome": "wrong", "reason": "predicted disposable_card_limits", "attempts": 1 }, { "row": "b77-53", "input": "Can I give my friends access to my account so they can top it up for me?", "expected": "topping_up_by_card", "got": "transfer_into_account", "outcome": "wrong", "reason": "predicted transfer_into_account", "attempts": 3 }, { "row": "b77-58", "input": "How do I know which ATMs will accept this card?", "expected": "atm_support", "got": "card_acceptance", "outcome": "wrong", "reason": "predicted card_acceptance", "attempts": 2 } ], "google/gemini-2.5-flash-lite": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-6", "input": "Why is the cash deposit not showing up in my account?", "expected": "balance_not_updated_after_cheque_or_cash_deposit", "got": "", "outcome": "error", "reason": "Provider returned error", "attempts": 3 }, { "row": "b77-7", "input": "How do I use the top up app with my apple watch?", "expected": "apple_pay_or_google_pay", "got": "", "outcome": "error", "reason": "Provider returned error", "attempts": 3 }, { "row": "b77-8", "input": "I need it delivered by Saturday.", "expected": "card_delivery_estimate", "got": "", "outcome": "error", "reason": "Provider returned error", "attempts": 3 }, { "row": "b77-9", "input": "It seems my top up is reverted?", "expected": "top_up_reverted", "got": "", "outcome": "error", "reason": "Provider returned error", "attempts": 3 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-10", "input": "How do I set a new pin?", "expected": "change_pin", "got": "", "outcome": "error", "reason": "Provider returned error", "attempts": 3 }, { "row": "b77-16", "input": "In what ATM can I change my pin?", "expected": "change_pin", "got": "atm_support", "outcome": "wrong", "reason": "predicted atm_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "", "outcome": "error", "reason": "Provider returned error", "attempts": 3 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 1 }, { "row": "b77-34", "input": "Any fee for topping up?", "expected": "top_up_by_card_charge", "got": "top_up_by_bank_transfer_charge", "outcome": "wrong", "reason": "predicted top_up_by_bank_transfer_charge", "attempts": 1 } ], "meta-llama/llama-3.3-70b-instruct": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "lost_or_stolen_card", "outcome": "wrong", "reason": "predicted lost_or_stolen_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-16", "input": "In what ATM can I change my pin?", "expected": "change_pin", "got": "atm_support", "outcome": "wrong", "reason": "predicted atm_support", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-28", "input": "Some of my card payments look different than where I purchased products from. Why is that?", "expected": "card_payment_not_recognised", "got": "declined_card_payment", "outcome": "wrong", "reason": "predicted declined_card_payment", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 1 }, { "row": "b77-38", "input": "I would appreciate that someone let me know when there is an extra charge for payments. I was checking the app earlier and saying a charge on one of my payments that was additional that no one made me aware of before.", "expected": "card_payment_fee_charged", "got": "extra_charge_on_statement", "outcome": "wrong", "reason": "predicted extra_charge_on_statement", "attempts": 1 }, { "row": "b77-42", "input": "When will the card arrive?", "expected": "card_delivery_estimate", "got": "card_arrival", "outcome": "wrong", "reason": "predicted card_arrival", "attempts": 1 }, { "row": "b77-47", "input": "how much do you charge to accept transfers", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-49", "input": "Can I also transfer with SWIFT?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_into_account", "outcome": "wrong", "reason": "predicted transfer_into_account", "attempts": 1 }, { "row": "b77-53", "input": "Can I give my friends access to my account so they can top it up for me?", "expected": "topping_up_by_card", "got": "beneficiary_not_allowed", "outcome": "wrong", "reason": "predicted beneficiary_not_allowed", "attempts": 1 } ], "mistralai/mistral-small-3.2-24b-instruct": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "lost_or_stolen_card", "outcome": "wrong", "reason": "predicted lost_or_stolen_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-16", "input": "In what ATM can I change my pin?", "expected": "change_pin", "got": "atm_support", "outcome": "wrong", "reason": "predicted atm_support", "attempts": 1 }, { "row": "b77-20", "input": "Can you let me know if Visa is among the card scheme assignments?", "expected": "visa_or_mastercard", "got": "supported_cards_and_currencies", "outcome": "wrong", "reason": "predicted supported_cards_and_currencies", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-23", "input": "I think my card payment had been return", "expected": "reverted_card_payment?", "got": "declined_card_payment", "outcome": "wrong", "reason": "predicted declined_card_payment", "attempts": 1 }, { "row": "b77-28", "input": "Some of my card payments look different than where I purchased products from. Why is that?", "expected": "card_payment_not_recognised", "got": "card_payment_wrong_exchange_rate", "outcome": "wrong", "reason": "predicted card_payment_wrong_exchange_rate", "attempts": 1 }, { "row": "b77-29", "input": "I need help. I asked for a refund from a merchant a while ago, but I have not gotten my money back. I keep checking my account, but nothing is showing up. What should I do now?", "expected": "Refund_not_showing_up", "got": "request_refund", "outcome": "wrong", "reason": "predicted request_refund", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 1 }, { "row": "b77-36", "input": "Is there a fee for transfer top-up?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-38", "input": "I would appreciate that someone let me know when there is an extra charge for payments. I was checking the app earlier and saying a charge on one of my payments that was additional that no one made me aware of before.", "expected": "card_payment_fee_charged", "got": "extra_charge_on_statement", "outcome": "wrong", "reason": "predicted extra_charge_on_statement", "attempts": 1 } ], "deepseek/deepseek-v4-flash": [ { "row": "b77-5", "input": "Do you accept other currencies besides US Dollars?", "expected": "supported_cards_and_currencies", "got": "fiat_currency_support", "outcome": "wrong", "reason": "predicted fiat_currency_support", "attempts": 1 }, { "row": "b77-11", "input": "Help! My wallet was stolen and someone is taking money out. I need this money! What can I do?", "expected": "cash_withdrawal_not_recognised", "got": "lost_or_stolen_card", "outcome": "wrong", "reason": "predicted lost_or_stolen_card", "attempts": 1 }, { "row": "b77-13", "input": "How long does a card delivery take?", "expected": "card_arrival", "got": "card_delivery_estimate", "outcome": "wrong", "reason": "predicted card_delivery_estimate", "attempts": 1 }, { "row": "b77-21", "input": "Can you tell me the total cost of a transfer?", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-23", "input": "I think my card payment had been return", "expected": "reverted_card_payment?", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 }, { "row": "b77-34", "input": "Any fee for topping up?", "expected": "top_up_by_card_charge", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 }, { "row": "b77-38", "input": "I would appreciate that someone let me know when there is an extra charge for payments. I was checking the app earlier and saying a charge on one of my payments that was additional that no one made me aware of before.", "expected": "card_payment_fee_charged", "got": "extra_charge_on_statement", "outcome": "wrong", "reason": "predicted extra_charge_on_statement", "attempts": 1 }, { "row": "b77-33", "input": "How do I set-up my PIN for the new card?", "expected": "get_physical_card", "got": "change_pin", "outcome": "wrong", "reason": "predicted change_pin", "attempts": 2 }, { "row": "b77-42", "input": "When will the card arrive?", "expected": "card_delivery_estimate", "got": "card_arrival", "outcome": "wrong", "reason": "predicted card_arrival", "attempts": 1 }, { "row": "b77-47", "input": "how much do you charge to accept transfers", "expected": "top_up_by_bank_transfer_charge", "got": "transfer_fee_charged", "outcome": "wrong", "reason": "predicted transfer_fee_charged", "attempts": 1 }, { "row": "b77-54", "input": "Are there restrictions for my disposable card since it does not seem to be working?", "expected": "virtual_card_not_working", "got": "disposable_card_limits", "outcome": "wrong", "reason": "predicted disposable_card_limits", "attempts": 1 }, { "row": "b77-49", "input": "Can I also transfer with SWIFT?", "expected": "top_up_by_bank_transfer_charge", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 } ] }, "run_cost_usd": 0.231512, "spend_cap_usd": 2, "capped": false, "cost_note": "cost = sum of OpenRouter usage.cost over every attempt, retries included" }