{ "schema": "clearbar.verdict.v1", "slug": "sms-spam", "title": "SMS spam detection (spam or legitimate message)", "task": "classification", "ran_at": "2026-11-05T02:07:00.790Z", "price_snapshot_at": "2026-11-05T02:07:00.784Z", "dataset": { "file": "data/sms-spam.jsonl", "name": "SMS Spam Collection (ucirvine/sms_spam), 5,574 labelled messages", "source": "https://huggingface.co/datasets/ucirvine/sms_spam", "licence": "CC BY 4.0 (UCI Machine Learning Repository)", "sample": "300 messages drawn with a fixed seed: 150 spam and 150 legitimate (ham)", "rows": 300 }, "threshold": 0.9, "margin": 0.05, "retries": 2, "temperature": 0, "max_tokens": 256, "system_prompt": "You decide whether a text message is spam. Choose the single label from the list that fits: spam for unsolicited promotions, prize or premium-rate scams and bulk marketing, ham for a normal personal or legitimate message. Reply with the label exactly as written and nothing else.", "baseline": "openai/gpt-6.1-sol", "models": [ { "model": "openai/gpt-6.1-sol", "is_baseline": true, "cells": 300, "successes": 295, "pass_rate": 0.9833, "wilson_lo": 0.9616, "wilson_hi": 0.9929, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "errors": 0, "cost_total": 0.079144, "cost_per_success": 0.000268, "cost_per_1k_tasks": 0.263813, "cost_estimated": false, "latency_p50_ms": 398, "latency_p95_ms": 604, "status": "cleared", "paired": null, "price": { "prompt_per_mtok": 2, "completion_per_mtok": 10 } }, { "model": "openai/gpt-6.1-terra", "is_baseline": false, "cells": 300, "successes": 295, "pass_rate": 0.9833, "wilson_lo": 0.9616, "wilson_hi": 0.9929, "attempts": 301, "attempts_per_task": 1.0033, "retries_total": 1, "retry_cost_share": 0.0064, "invalid_final": 0, "errors": 0, "cost_total": 0.048089, "cost_per_success": 0.000163, "cost_per_1k_tasks": 0.160297, "cost_estimated": false, "latency_p50_ms": 402, "latency_p95_ms": 599, "status": "cleared", "paired": { "b": 0, "c": 0, "n": 300, "diff": 0, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 1, "completion_per_mtok": 5 } }, { "model": "openai/gpt-6-luna", "is_baseline": false, "cells": 300, "successes": 291, "pass_rate": 0.97, "wilson_lo": 0.944, "wilson_hi": 0.9841, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "errors": 0, "cost_total": 0.007537, "cost_per_success": 0.000026, "cost_per_1k_tasks": 0.025124, "cost_estimated": false, "latency_p50_ms": 387, "latency_p95_ms": 604, "status": "cleared", "paired": { "b": 5, "c": 1, "n": 300, "diff": -0.0133, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.5 } }, { "model": "anthropic/claude-haiku-5.5", "is_baseline": false, "cells": 300, "successes": 292, "pass_rate": 0.9733, "wilson_lo": 0.9483, "wilson_hi": 0.9864, "attempts": 303, "attempts_per_task": 1.01, "retries_total": 3, "retry_cost_share": 0.06, "invalid_final": 1, "errors": 0, "cost_total": 0.0061, "cost_per_success": 0.000021, "cost_per_1k_tasks": 0.020332, "cost_estimated": false, "latency_p50_ms": 250, "latency_p95_ms": 330, "status": "cleared", "paired": { "b": 3, "c": 0, "n": 300, "diff": -0.01, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.5 } }, { "model": "google/gemini-3.5-flash-lite", "is_baseline": false, "cells": 300, "successes": 292, "pass_rate": 0.9733, "wilson_lo": 0.9483, "wilson_hi": 0.9864, "attempts": 302, "attempts_per_task": 1.0067, "retries_total": 2, "retry_cost_share": 0, "invalid_final": 1, "errors": 0, "cost_total": 0.009251, "cost_per_success": 0.000032, "cost_per_1k_tasks": 0.030837, "cost_estimated": true, "latency_p50_ms": 375, "latency_p95_ms": 591, "status": "cleared", "paired": { "b": 6, "c": 3, "n": 300, "diff": -0.01, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.3, "completion_per_mtok": 2.5 } }, { "model": "deepseek/deepseek-v4.1-flash", "is_baseline": false, "cells": 300, "successes": 289, "pass_rate": 0.9633, "wilson_lo": 0.9355, "wilson_hi": 0.9794, "attempts": 311, "attempts_per_task": 1.0367, "retries_total": 11, "retry_cost_share": 0.0819, "invalid_final": 3, "errors": 0, "cost_total": 0.014596, "cost_per_success": 0.000051, "cost_per_1k_tasks": 0.048653, "cost_estimated": false, "latency_p50_ms": 344, "latency_p95_ms": 1042, "status": "cleared", "paired": { "b": 6, "c": 0, "n": 300, "diff": -0.02, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.045, "completion_per_mtok": 0.5 } }, { "model": "qwen/qwen3.8-flash", "is_baseline": false, "cells": 300, "successes": 293, "pass_rate": 0.9767, "wilson_lo": 0.9526, "wilson_hi": 0.9887, "attempts": 333, "attempts_per_task": 1.11, "retries_total": 33, "retry_cost_share": 0.1219, "invalid_final": 0, "errors": 0, "cost_total": 0.021921, "cost_per_success": 0.000075, "cost_per_1k_tasks": 0.073068, "cost_estimated": false, "latency_p50_ms": 815, "latency_p95_ms": 6345, "status": "cleared", "paired": { "b": 3, "c": 1, "n": 300, "diff": -0.0067, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.15, "completion_per_mtok": 0.47 } }, { "model": "mistralai/mistral-small-2603", "is_baseline": false, "cells": 300, "successes": 282, "pass_rate": 0.94, "wilson_lo": 0.9072, "wilson_hi": 0.9617, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "errors": 0, "cost_total": 0.004509, "cost_per_success": 0.000016, "cost_per_1k_tasks": 0.015031, "cost_estimated": false, "latency_p50_ms": 338, "latency_p95_ms": 826, "status": "cleared", "paired": { "b": 16, "c": 3, "n": 300, "diff": -0.0433, "p": 0.323045, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.15, "completion_per_mtok": 0.6 } }, { "model": "meta-llama/llama-4-maverick", "is_baseline": false, "cells": 300, "successes": 283, "pass_rate": 0.9433, "wilson_lo": 0.9111, "wilson_hi": 0.9643, "attempts": 315, "attempts_per_task": 1.05, "retries_total": 15, "retry_cost_share": 0.0319, "invalid_final": 0, "errors": 0, "cost_total": 0.007511, "cost_per_success": 0.000027, "cost_per_1k_tasks": 0.025038, "cost_estimated": false, "latency_p50_ms": 640, "latency_p95_ms": 1187, "status": "cleared", "paired": { "b": 15, "c": 3, "n": 300, "diff": -0.04, "p": 0.185366, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.35, "completion_per_mtok": 1 } } ], "verdict": { "kind": "switch", "model": "anthropic/claude-haiku-5.5", "matches_baseline": true, "cost_per_success": 0.000021, "baseline_cost_per_success": 0.000268, "saving_pct": 0.9216 }, "payback": [ { "monthly_bill": 200, "monthly_saving": 184.33, "days_to_repay": [ { "price": 19, "days": 3.1 }, { "price": 49, "days": 8 } ] }, { "monthly_bill": 500, "monthly_saving": 460.82, "days_to_repay": [ { "price": 19, "days": 1.2 }, { "price": 49, "days": 3.2 } ] }, { "monthly_bill": 2000, "monthly_saving": 1843.28, "days_to_repay": [ { "price": 19, "days": 0.3 }, { "price": 49, "days": 0.8 } ] } ], "failures": { "openai/gpt-6.1-sol": [ { "row": "sm-57", "input": "Latest News! Police station toilet stolen, cops have nothing to go on!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-79", "input": "thesmszone.com lets you send free anonymous and masked messages..im sending this message from there..do you see the potential for abuse???", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 } ], "openai/gpt-6.1-terra": [ { "row": "sm-57", "input": "Latest News! Police station toilet stolen, cops have nothing to go on!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-79", "input": "thesmszone.com lets you send free anonymous and masked messages..im sending this message from there..do you see the potential for abuse???", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 } ], "openai/gpt-6-luna": [ { "row": "sm-1", "input": "In The Simpsons Movie released in July 2007 name the band that died at the start of the film? A-Green Day, B-Blue Day, C-Red Day. (Send A, B or C)", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-57", "input": "Latest News! Police station toilet stolen, cops have nothing to go on!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-90", "input": "Missed call alert. These numbers called but left no message. 07008009200", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-102", "input": "Macha dont feel upset.i can assume your mindset.believe me one evening with me and i have some wonderful plans for both of us.LET LIFE BEGIN AGAIN.call me anytime", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-206", "input": "I always chat with you. In fact i need money can you raise me?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-217", "input": "Have you laid your airtel line to rest?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 } ], "anthropic/claude-haiku-5.5": [ { "row": "sm-1", "input": "In The Simpsons Movie released in July 2007 name the band that died at the start of the film? A-Green Day, B-Blue Day, C-Red Day. (Send A, B or C)", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 2 }, { "row": "sm-57", "input": "Latest News! Police station toilet stolen, cops have nothing to go on!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-79", "input": "thesmszone.com lets you send free anonymous and masked messages..im sending this message from there..do you see the potential for abuse???", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-166", "input": "Let Ur Heart Be Ur Compass Ur Mind Ur Map Ur Soul Ur Guide And U Will Never loose in world....gnun - Sent via WAY2SMS.COM", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-238", "input": "No let me do the math. Your not good at it.", "expected": "ham", "got": "I don't see a message to classify yet. Please paste the text you want checked, and I'll reply with either \"ham\" or \"spam.\"", "outcome": "invalid", "reason": "label outside the set: I dont see a message to classify yet. Please paste the text", "attempts": 3 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 } ], "google/gemini-3.5-flash-lite": [ { "row": "sm-61", "input": "Motivate Behind every darkness, there is a shining light waiting for you to find it... Behind every best friend, there is always trust and love... BSLVYL", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-102", "input": "Macha dont feel upset.i can assume your mindset.believe me one evening with me and i have some wonderful plans for both of us.LET LIFE BEGIN AGAIN.call me anytime", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-166", "input": "Let Ur Heart Be Ur Compass Ur Mind Ur Map Ur Soul Ur Guide And U Will Never loose in world....gnun - Sent via WAY2SMS.COM", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-206", "input": "I always chat with you. In fact i need money can you raise me?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-217", "input": "Have you laid your airtel line to rest?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-285", "input": "Waaaat?? Lololo ok next time then!", "expected": "ham", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 } ], "deepseek/deepseek-v4.1-flash": [ { "row": "sm-1", "input": "In The Simpsons Movie released in July 2007 name the band that died at the start of the film? A-Green Day, B-Blue Day, C-Red Day. (Send A, B or C)", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-28", "input": "Also maaaan are you missing out", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 2 }, { "row": "sm-57", "input": "Latest News! Police station toilet stolen, cops have nothing to go on!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-61", "input": "Motivate Behind every darkness, there is a shining light waiting for you to find it... Behind every best friend, there is always trust and love... BSLVYL", "expected": "ham", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 }, { "row": "sm-79", "input": "thesmszone.com lets you send free anonymous and masked messages..im sending this message from there..do you see the potential for abuse???", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-112", "input": "I keep ten rs in my shelf:) buy two egg.", "expected": "ham", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-166", "input": "Let Ur Heart Be Ur Compass Ur Mind Ur Map Ur Soul Ur Guide And U Will Never loose in world....gnun - Sent via WAY2SMS.COM", "expected": "ham", "got": "", "outcome": "invalid", "reason": "empty output", "attempts": 3 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-217", "input": "Have you laid your airtel line to rest?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 3 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 } ], "qwen/qwen3.8-flash": [ { "row": "sm-1", "input": "In The Simpsons Movie released in July 2007 name the band that died at the start of the film? A-Green Day, B-Blue Day, C-Red Day. (Send A, B or C)", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 3 }, { "row": "sm-57", "input": "Latest News! Police station toilet stolen, cops have nothing to go on!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-166", "input": "Let Ur Heart Be Ur Compass Ur Mind Ur Map Ur Soul Ur Guide And U Will Never loose in world....gnun - Sent via WAY2SMS.COM", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-212", "input": "Helloooo... Wake up..! \"Sweet\" \"morning\" \"welcomes\" \"You\" \"Enjoy\" \"This Day\" \"with full of joy\".. \"GUD MRNG\".", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-258", "input": "Do you ever notice that when you're driving, anyone going slower than you is an idiot and everyone driving faster than you is a maniac?", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 } ], "mistralai/mistral-small-2603": [ { "row": "sm-24", "input": "Don't Think About \"What u Have Got\" Think About \"How to Use It That You Have Got\" gooD ni8", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-61", "input": "Motivate Behind every darkness, there is a shining light waiting for you to find it... Behind every best friend, there is always trust and love... BSLVYL", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-67", "input": "Where are you lover ? I need you ...", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-78", "input": "My Parents, My Kidz, My Friends n My Colleagues. All screaming.. SURPRISE !! and I was waiting on the sofa.. ... ..... ' NAKED...!", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-84", "input": "Come to mahal bus stop.. <DECIMAL>", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-95", "input": "Very hurting n meaningful lines ever: \"I compromised everything for my love, But at d end my love compromised me for everything:-(\".. Gud mornin:-)", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-102", "input": "Macha dont feel upset.i can assume your mindset.believe me one evening with me and i have some wonderful plans for both of us.LET LIFE BEGIN AGAIN.call me anytime", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-105", "input": "I doubt you could handle 5 times per night in any case...", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-120", "input": "ASKED 3MOBILE IF 0870 CHATLINES INCLU IN FREE MINS. INDIA CUST SERVs SED YES. L8ER GOT MEGA BILL. 3 DONT GIV A SHIT. BAILIFF DUE IN DAYS. I O £250 3 WANT £800", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-166", "input": "Let Ur Heart Be Ur Compass Ur Mind Ur Map Ur Soul Ur Guide And U Will Never loose in world....gnun - Sent via WAY2SMS.COM", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-204", "input": "Meeting u is my work. . . Tel me when shall i do my work tomorrow", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-206", "input": "I always chat with you. In fact i need money can you raise me?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 } ], "meta-llama/llama-4-maverick": [ { "row": "sm-1", "input": "In The Simpsons Movie released in July 2007 name the band that died at the start of the film? A-Green Day, B-Blue Day, C-Red Day. (Send A, B or C)", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-28", "input": "Also maaaan are you missing out", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-61", "input": "Motivate Behind every darkness, there is a shining light waiting for you to find it... Behind every best friend, there is always trust and love... BSLVYL", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-78", "input": "My Parents, My Kidz, My Friends n My Colleagues. All screaming.. SURPRISE !! and I was waiting on the sofa.. ... ..... ' NAKED...!", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-80", "input": "I hope your pee burns tonite.", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-90", "input": "Missed call alert. These numbers called but left no message. 07008009200", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-102", "input": "Macha dont feel upset.i can assume your mindset.believe me one evening with me and i have some wonderful plans for both of us.LET LIFE BEGIN AGAIN.call me anytime", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-105", "input": "I doubt you could handle 5 times per night in any case...", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-113", "input": "Or just do that 6times", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-166", "input": "Let Ur Heart Be Ur Compass Ur Mind Ur Map Ur Soul Ur Guide And U Will Never loose in world....gnun - Sent via WAY2SMS.COM", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 }, { "row": "sm-176", "input": "Did you hear about the new \"Divorce Barbie\"? It comes with all of Ken's stuff!", "expected": "spam", "got": "ham", "outcome": "wrong", "reason": "predicted ham", "attempts": 1 }, { "row": "sm-206", "input": "I always chat with you. In fact i need money can you raise me?", "expected": "ham", "got": "spam", "outcome": "wrong", "reason": "predicted spam", "attempts": 1 } ] }, "run_cost_usd": 0.198658, "spend_cap_usd": 1, "capped": false, "cost_note": "cost = sum of OpenRouter usage.cost over every attempt, retries included" }