{ "schema": "clearbar.verdict.v1", "slug": "glaive-tool-call", "title": "Tool-call argument checking: pick the right function and fill its arguments", "short_title": "Tool-call arguments", "task": "tool-call", "ran_at": "2026-11-05T02:38:51.193Z", "price_snapshot_at": "2026-11-05T02:38:51.191Z", "dataset": { "file": "data/glaive-tool-call.jsonl", "name": "glaive-function-calling-v2 (glaiveai/glaive-function-calling-v2)", "source": "https://huggingface.co/datasets/glaiveai/glaive-function-calling-v2", "licence": "Apache-2.0", "sample": "300 rows drawn with a fixed seed from the eligible rows found in 36 random pages (3,600 rows) of the 112,960-row train split, at most 5 per function; eligible = every argument value is stated in the conversation", "rows": 300 }, "threshold": 0.85, "margin": 0.05, "retries": 2, "temperature": 0, "max_tokens": 400, "system_prompt": "You are given tool definitions and a conversation. Reply with the one tool call the assistant should make next, as a single JSON object {\"name\": \"\", \"arguments\": {: }} and nothing else. Use only the tools and parameters declared, and fill each value from what the user said.", "baseline": "openai/gpt-6.1-sol", "models": [ { "model": "openai/gpt-6.1-sol", "is_baseline": true, "cells": 300, "successes": 282, "pass_rate": 0.94, "wilson_lo": 0.9072, "wilson_hi": 0.9617, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "failure_kinds": { "wording": 13, "number_format": 3, "missing_argument": 2 }, "errors": 0, "cost_total": 0.216948, "cost_per_success": 0.000769, "cost_per_1k_tasks": 0.72316, "cost_estimated": false, "latency_p50_ms": 414, "latency_p95_ms": 752, "status": "cleared", "paired": null, "price": { "prompt_per_mtok": 2, "completion_per_mtok": 10 } }, { "model": "openai/gpt-6.1-terra", "is_baseline": false, "cells": 300, "successes": 281, "pass_rate": 0.9367, "wilson_lo": 0.9032, "wilson_hi": 0.9591, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "failure_kinds": { "wording": 11, "number_format": 4, "different_value": 2, "missing_argument": 2 }, "errors": 0, "cost_total": 0.119449, "cost_per_success": 0.000425, "cost_per_1k_tasks": 0.398163, "cost_estimated": false, "latency_p50_ms": 422, "latency_p95_ms": 740, "status": "cleared", "paired": { "b": 2, "c": 1, "n": 300, "diff": -0.0033, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 1, "completion_per_mtok": 5 } }, { "model": "openai/gpt-6-luna", "is_baseline": false, "cells": 300, "successes": 279, "pass_rate": 0.93, "wilson_lo": 0.8954, "wilson_hi": 0.9538, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "failure_kinds": { "wording": 16, "different_value": 2, "number_format": 2, "missing_argument": 1 }, "errors": 0, "cost_total": 0.014118, "cost_per_success": 0.000051, "cost_per_1k_tasks": 0.047061, "cost_estimated": false, "latency_p50_ms": 406, "latency_p95_ms": 773, "status": "cleared", "paired": { "b": 5, "c": 2, "n": 300, "diff": -0.01, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.5 } }, { "model": "anthropic/claude-haiku-5.5", "is_baseline": false, "cells": 300, "successes": 284, "pass_rate": 0.9467, "wilson_lo": 0.9151, "wilson_hi": 0.9669, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "failure_kinds": { "wording": 11, "different_value": 2, "missing_argument": 2, "number_format": 1 }, "errors": 0, "cost_total": 0.018125, "cost_per_success": 0.000064, "cost_per_1k_tasks": 0.060416, "cost_estimated": false, "latency_p50_ms": 248, "latency_p95_ms": 372, "status": "cleared", "paired": { "b": 4, "c": 6, "n": 300, "diff": 0.0067, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.1, "completion_per_mtok": 0.5 } }, { "model": "google/gemini-3.5-flash-lite", "is_baseline": false, "cells": 300, "successes": 285, "pass_rate": 0.95, "wilson_lo": 0.9192, "wilson_hi": 0.9695, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "failure_kinds": { "wording": 9, "number_format": 3, "missing_argument": 2, "different_value": 1 }, "errors": 0, "cost_total": 0.044771, "cost_per_success": 0.000157, "cost_per_1k_tasks": 0.149237, "cost_estimated": false, "latency_p50_ms": 391, "latency_p95_ms": 614, "status": "cleared", "paired": { "b": 3, "c": 6, "n": 300, "diff": 0.01, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.3, "completion_per_mtok": 2.5 } }, { "model": "deepseek/deepseek-v4.1-flash", "is_baseline": false, "cells": 300, "successes": 288, "pass_rate": 0.96, "wilson_lo": 0.9314, "wilson_hi": 0.977, "attempts": 302, "attempts_per_task": 1.0067, "retries_total": 2, "retry_cost_share": 0.0173, "invalid_final": 0, "failure_kinds": { "wording": 11, "missing_argument": 1 }, "errors": 0, "cost_total": 0.020693, "cost_per_success": 0.000072, "cost_per_1k_tasks": 0.068976, "cost_estimated": false, "latency_p50_ms": 368, "latency_p95_ms": 1159, "status": "cleared", "paired": { "b": 2, "c": 8, "n": 300, "diff": 0.02, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.045, "completion_per_mtok": 0.5 } }, { "model": "qwen/qwen3.8-flash", "is_baseline": false, "cells": 300, "successes": 279, "pass_rate": 0.93, "wilson_lo": 0.8954, "wilson_hi": 0.9538, "attempts": 305, "attempts_per_task": 1.0167, "retries_total": 5, "retry_cost_share": 0.0194, "invalid_final": 0, "failure_kinds": { "wording": 12, "different_value": 4, "number_format": 3, "missing_argument": 2 }, "errors": 0, "cost_total": 0.025824, "cost_per_success": 0.000093, "cost_per_1k_tasks": 0.086079, "cost_estimated": false, "latency_p50_ms": 803, "latency_p95_ms": 5838, "status": "cleared", "paired": { "b": 4, "c": 1, "n": 300, "diff": -0.01, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.15, "completion_per_mtok": 0.47 } }, { "model": "mistralai/mistral-small-2603", "is_baseline": false, "cells": 300, "successes": 212, "pass_rate": 0.7067, "wilson_lo": 0.6528, "wilson_hi": 0.7553, "attempts": 434, "attempts_per_task": 1.4467, "retries_total": 134, "retry_cost_share": 0.2391, "invalid_final": 58, "failure_kinds": { "empty_reply": 58, "wording": 14, "different_value": 10, "number_format": 5, "missing_argument": 1 }, "errors": 0, "cost_total": 0.020673, "cost_per_success": 0.000098, "cost_per_1k_tasks": 0.068909, "cost_estimated": false, "latency_p50_ms": 341, "latency_p95_ms": 1757, "status": "below bar", "paired": { "b": 74, "c": 4, "n": 300, "diff": -0.2333, "p": 1, "non_inferior": false, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.15, "completion_per_mtok": 0.6 } }, { "model": "meta-llama/llama-4-maverick", "is_baseline": false, "cells": 300, "successes": 284, "pass_rate": 0.9467, "wilson_lo": 0.9151, "wilson_hi": 0.9669, "attempts": 300, "attempts_per_task": 1, "retries_total": 0, "retry_cost_share": 0, "invalid_final": 0, "failure_kinds": { "wording": 13, "number_format": 2, "different_value": 1 }, "errors": 0, "cost_total": 0.022583, "cost_per_success": 0.00008, "cost_per_1k_tasks": 0.075276, "cost_estimated": false, "latency_p50_ms": 583, "latency_p95_ms": 1008, "status": "cleared", "paired": { "b": 6, "c": 8, "n": 300, "diff": 0.0067, "p": 0, "non_inferior": true, "margin": 0.05 }, "price": { "prompt_per_mtok": 0.35, "completion_per_mtok": 1 } } ], "verdict": { "kind": "switch", "model": "openai/gpt-6-luna", "matches_baseline": true, "cost_per_success": 0.000051, "baseline_cost_per_success": 0.000769, "saving_pct": 0.9337 }, "payback": [ { "monthly_bill": 200, "monthly_saving": 186.74, "days_to_repay": [ { "price": 19, "days": 3.1 }, { "price": 49, "days": 7.9 } ] }, { "monthly_bill": 500, "monthly_saving": 466.84, "days_to_repay": [ { "price": 19, "days": 1.2 }, { "price": 49, "days": 3.1 } ] }, { "monthly_bill": 2000, "monthly_saving": 1867.36, "days_to_repay": [ { "price": 19, "days": 0.3 }, { "price": 49, "days": 0.8 } ] } ], "failures": { "openai/gpt-6.1-sol": [ { "row": "tc-52", "input": "USER: Hi, I need help with calculating my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-75", "input": "USER: I want to watch a movie tonight. Can you suggest me some good ones? ASSISTANT: Sure, I can help with that. Can you tell me your preferences? Like any specific genre, actor or year of release? USER: I like action movies. And I prefer movies released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\",\"year\":2010}}", "got": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action movies released after 2010\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: search_query, year", "attempts": 1 }, { "row": "tc-104", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee.", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a\",\"author\":\"Harper Lee\"}}", "got": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a...\",\"author\":\"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-103", "input": "USER: Hi, I need to calculate my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"Birthday party\",\"date\":\"2022-05-15\",\"location\":\"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-120", "input": "USER: Hi, I had a cheeseburger for lunch today. It was about 563 calories. Can you log this for me? ASSISTANT: Sure, I can help with that. Let me log this information for you.", "expected": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger\",\"calories\":563,\"date\":\"today\"}}", "got": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger for lunch\",\"calories\":563,\"date\":\"today\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: meal", "attempts": 1 }, { "row": "tc-122", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the details of the meeting such as the name of the event, start time, end time, and location? USER: The meeting is called \"Project Discussion\". It will start at 10:00 AM and end at 11:30 AM. The loca …", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\",\"location\":\"Conference Room\"}}", "got": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"tomorrow at 10:00 AM\",\"end_time\":\"tomorrow at 11:30 AM\",\"location\":\"Conference Room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"tomorrow at 10:00 AM\",\"end_time\":\"tomorrow at 11:00 AM\",\"location\":\"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-130", "input": "USER: I'm in the mood for Italian food. Can you find any Italian restaurants near me?", "expected": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\",\"location\":\"near me\"}}", "got": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: location", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"French\",\"target_language\":\"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-177", "input": "USER: I need to create an invoice for a customer named John Doe. He bought 2 apples for $1 each and 3 bananas for $0.5 each. The tax rate is 10%.", "expected": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apple\",\"price\":1,\"quantity\":2},{\"name\":\"banana\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "got": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apples\",\"price\":1,\"quantity\":2},{\"name\":\"bananas\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: items", "attempts": 1 }, { "row": "tc-178", "input": "USER: Hey, I need to calculate my carbon footprint for driving my car for 5 hours. Can you help me with that? ASSISTANT: Sure, I can help you with that. Let me calculate it for you.", "expected": "{\"name\":\"calculate_carbon_footprint\",\"arguments\":{\"activity\":\"driving\",\"duration\":5}}", "got": "{\"name\":\"calculate_carbon_footprint\",\"arguments\":{\"activity\":\"driving my car\",\"duration\":5}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: activity", "attempts": 1 } ], "openai/gpt-6.1-terra": [ { "row": "tc-10", "input": "USER: I want to watch a movie. Can you suggest me some? ASSISTANT: Of course! Could you please tell me your preferred genre and the year of release you're interested in? USER: I like action movies. I would prefer something released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2010}}", "got": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2011}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-52", "input": "USER: Hi, I need help with calculating my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-75", "input": "USER: I want to watch a movie tonight. Can you suggest me some good ones? ASSISTANT: Sure, I can help with that. Can you tell me your preferences? Like any specific genre, actor or year of release? USER: I like action movies. And I prefer movies released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\",\"year\":2010}}", "got": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-103", "input": "USER: Hi, I need to calculate my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-104", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee.", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a\",\"author\":\"Harper Lee\"}}", "got": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a...\",\"author\":\"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"Birthday party\",\"date\":\"2022-05-15\",\"location\":\"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-120", "input": "USER: Hi, I had a cheeseburger for lunch today. It was about 563 calories. Can you log this for me? ASSISTANT: Sure, I can help with that. Let me log this information for you.", "expected": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger\",\"calories\":563,\"date\":\"today\"}}", "got": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger (lunch)\",\"calories\":563,\"date\":\"2026-07-17\"}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: meal, date", "attempts": 1 }, { "row": "tc-122", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the details of the meeting such as the name of the event, start time, end time, and location? USER: The meeting is called \"Project Discussion\". It will start at 10:00 AM and end at 11:30 AM. The loca …", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\",\"location\":\"Conference Room\"}}", "got": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"tomorrow at 10:00 AM\",\"end_time\":\"tomorrow at 11:30 AM\",\"location\":\"Conference Room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM tomorrow\",\"end_time\":\"11:00 AM tomorrow\",\"location\":\"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-130", "input": "USER: I'm in the mood for Italian food. Can you find any Italian restaurants near me?", "expected": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\",\"location\":\"near me\"}}", "got": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: location", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"French\",\"target_language\":\"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-171", "input": "USER: Hi, I need to calculate the interest on my savings. I have $5000 in my account and the interest rate is 2% per year. I plan to keep the money in the account for 3 years. Can you help me calculate the interest?", "expected": "{\"name\":\"calculate_interest\",\"arguments\":{\"principal\":5000,\"rate\":2,\"time\":3}}", "got": "{\"name\":\"calculate_interest\",\"arguments\":{\"principal\":5000,\"rate\":0.02,\"time\":3}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: rate", "attempts": 1 } ], "openai/gpt-6-luna": [ { "row": "tc-10", "input": "USER: I want to watch a movie. Can you suggest me some? ASSISTANT: Of course! Could you please tell me your preferred genre and the year of release you're interested in? USER: I like action movies. I would prefer something released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2010}}", "got": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2011}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-43", "input": "USER: I am looking for a book but I can't remember the title. The author's name is George Orwell and it's a dystopian novel.", "expected": "{\"name\":\"search_books\",\"arguments\":{\"author\":\"George Orwell\",\"genre\":\"dystopian\"}}", "got": "{\"name\":\"search_books\",\"arguments\":{\"author\":\"George Orwell\",\"genre\":\"dystopian novel\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: genre", "attempts": 1 }, { "row": "tc-52", "input": "USER: Hi, I need help with calculating my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-75", "input": "USER: I want to watch a movie tonight. Can you suggest me some good ones? ASSISTANT: Sure, I can help with that. Can you tell me your preferences? Like any specific genre, actor or year of release? USER: I like action movies. And I prefer movies released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\",\"year\":2010}}", "got": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action movies released after 2010\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: search_query, year", "attempts": 1 }, { "row": "tc-104", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee.", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a\",\"author\":\"Harper Lee\"}}", "got": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a Mockingbird\",\"author\":\"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-119", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee. Can you help me find it?", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a...\",\"author\":\"Harper Lee\"}}", "got": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a Mockingbird\",\"author\":\"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-120", "input": "USER: Hi, I had a cheeseburger for lunch today. It was about 563 calories. Can you log this for me? ASSISTANT: Sure, I can help with that. Let me log this information for you.", "expected": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger\",\"calories\":563,\"date\":\"today\"}}", "got": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"Cheeseburger for lunch\",\"calories\":563,\"date\":\"today\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: meal", "attempts": 1 }, { "row": "tc-122", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the details of the meeting such as the name of the event, start time, end time, and location? USER: The meeting is called \"Project Discussion\". It will start at 10:00 AM and end at 11:30 AM. The loca …", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\",\"location\":\"Conference Room\"}}", "got": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"tomorrow at 10:00 AM\",\"end_time\":\"tomorrow at 11:30 AM\",\"location\":\"Conference Room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"tomorrow 10:00 AM\",\"end_time\":\"tomorrow 11:00 AM\",\"location\":\"the conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time, location", "attempts": 1 }, { "row": "tc-162", "input": "USER: I am looking for a book but I can't remember the exact title. It's something about a mockingbird.", "expected": "{\"name\":\"search_books\",\"arguments\":{\"query\":\"mockingbird\"}}", "got": "{\"name\":\"search_books\",\"arguments\":{\"query\":\"book about a mockingbird\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: query", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"French\",\"target_language\":\"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 } ], "anthropic/claude-haiku-5.5": [ { "row": "tc-10", "input": "USER: I want to watch a movie. Can you suggest me some? ASSISTANT: Of course! Could you please tell me your preferred genre and the year of release you're interested in? USER: I like action movies. I would prefer something released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2010}}", "got": "{\"name\": \"search_movies\", \"arguments\": {\"genre\": \"action\", \"year\": 2011}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-75", "input": "USER: I want to watch a movie tonight. Can you suggest me some good ones? ASSISTANT: Sure, I can help with that. Can you tell me your preferences? Like any specific genre, actor or year of release? USER: I like action movies. And I prefer movies released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\",\"year\":2010}}", "got": "{\"name\": \"search_movies\", \"arguments\": {\"search_query\": \"action\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\": \"create_event\", \"arguments\": {\"title\": \"Birthday Party\", \"date\": \"2022-05-15\", \"location\": \"My house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-119", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee. Can you help me find it?", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a...\",\"author\":\"Harper Lee\"}}", "got": "{\"name\": \"search_book\", \"arguments\": {\"title\": \"To Kill a\", \"author\": \"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-120", "input": "USER: Hi, I had a cheeseburger for lunch today. It was about 563 calories. Can you log this for me? ASSISTANT: Sure, I can help with that. Let me log this information for you.", "expected": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger\",\"calories\":563,\"date\":\"today\"}}", "got": "{\"name\": \"track_calories\", \"arguments\": {\"meal\": \"Cheeseburger (lunch)\", \"calories\": 563, \"date\": \"today\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: meal", "attempts": 1 }, { "row": "tc-122", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the details of the meeting such as the name of the event, start time, end time, and location? USER: The meeting is called \"Project Discussion\". It will start at 10:00 AM and end at 11:30 AM. The loca …", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\",\"location\":\"Conference Room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_name\": \"Project Discussion\", \"start_time\": \"tomorrow at 10:00 AM\", \"end_time\": \"tomorrow at 11:30 AM\", \"location\": \"Conference Room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_title\": \"Team Meeting\", \"start_time\": \"10:00 AM tomorrow\", \"end_time\": \"11:00 AM tomorrow\", \"location\": \"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-130", "input": "USER: I'm in the mood for Italian food. Can you find any Italian restaurants near me?", "expected": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\",\"location\":\"near me\"}}", "got": "{\"name\": \"search_restaurants\", \"arguments\": {\"cuisine\": \"Italian\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: location", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\": \"translate_text\", \"arguments\": {\"text\": \"Je suis très heureux de vous rencontrer\", \"source_language\": \"French\", \"target_language\": \"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-177", "input": "USER: I need to create an invoice for a customer named John Doe. He bought 2 apples for $1 each and 3 bananas for $0.5 each. The tax rate is 10%.", "expected": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apple\",\"price\":1,\"quantity\":2},{\"name\":\"banana\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "got": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apples\",\"price\":1,\"quantity\":2},{\"name\":\"bananas\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: items", "attempts": 1 }, { "row": "tc-178", "input": "USER: Hey, I need to calculate my carbon footprint for driving my car for 5 hours. Can you help me with that? ASSISTANT: Sure, I can help you with that. Let me calculate it for you.", "expected": "{\"name\":\"calculate_carbon_footprint\",\"arguments\":{\"activity\":\"driving\",\"duration\":5}}", "got": "{\"name\": \"calculate_carbon_footprint\", \"arguments\": {\"activity\": \"driving a car\", \"duration\": 5}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: activity", "attempts": 1 }, { "row": "tc-212", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, and end time for the meeting? USER: The meeting is titled \"Project Discussion\". It should start at 10:00 AM and end at 11:30 AM.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"title\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"title\": \"Project Discussion\", \"start_time\": \"tomorrow at 10:00 AM\", \"end_time\": \"tomorrow at 11:30 AM\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 } ], "google/gemini-3.5-flash-lite": [ { "row": "tc-75", "input": "USER: I want to watch a movie tonight. Can you suggest me some good ones? ASSISTANT: Sure, I can help with that. Can you tell me your preferences? Like any specific genre, actor or year of release? USER: I like action movies. And I prefer movies released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\",\"year\":2010}}", "got": "{\"name\": \"search_movies\", \"arguments\": {\"search_query\": \"action\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-103", "input": "USER: Hi, I need to calculate my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\": \"calculate_tax\", \"arguments\": {\"income\": 50000, \"tax_rate\": 0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\": \"create_event\", \"arguments\": {\"title\": \"Birthday Party\", \"date\": \"2022-05-15\", \"location\": \"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-119", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee. Can you help me find it?", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a...\",\"author\":\"Harper Lee\"}}", "got": "{\"name\": \"search_book\", \"arguments\": {\"author\": \"Harper Lee\", \"title\": \"To Kill a\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_title\": \"Team Meeting\", \"start_time\": \"tomorrow at 10:00 AM\", \"end_time\": \"tomorrow at 11:00 AM\", \"location\": \"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-130", "input": "USER: I'm in the mood for Italian food. Can you find any Italian restaurants near me?", "expected": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\",\"location\":\"near me\"}}", "got": "{\"name\": \"search_restaurants\", \"arguments\": {\"cuisine\": \"Italian\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: location", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\": \"translate_text\", \"arguments\": {\"text\": \"Je suis tr\\u00e8s heureux de vous rencontrer\", \"source_language\": \"French\", \"target_language\": \"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-171", "input": "USER: Hi, I need to calculate the interest on my savings. I have $5000 in my account and the interest rate is 2% per year. I plan to keep the money in the account for 3 years. Can you help me calculate the interest?", "expected": "{\"name\":\"calculate_interest\",\"arguments\":{\"principal\":5000,\"rate\":2,\"time\":3}}", "got": "{\"name\": \"calculate_interest\", \"arguments\": {\"principal\": 5000, \"rate\": 0.02, \"time\": 3}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: rate", "attempts": 1 }, { "row": "tc-177", "input": "USER: I need to create an invoice for a customer named John Doe. He bought 2 apples for $1 each and 3 bananas for $0.5 each. The tax rate is 10%.", "expected": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apple\",\"price\":1,\"quantity\":2},{\"name\":\"banana\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "got": "{\"name\": \"create_invoice\", \"arguments\": {\"customer_name\": \"John Doe\", \"items\": [{\"name\": \"apples\", \"price\": 1.0, \"quantity\": 2}, {\"name\": \"bananas\", \"price\": 0.5, \"quantity\": 3}], \"tax_rate\": 10.0}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: items", "attempts": 1 }, { "row": "tc-200", "input": "USER: I am currently in New York and I am looking for nearby restaurants within a 500 meter radius. Can you help me with that?", "expected": "{\"name\":\"find_nearby_places\",\"arguments\":{\"location\":\"New York\",\"radius\":500,\"type\":\"restaurant\"}}", "got": "{\"name\": \"find_nearby_places\", \"arguments\": {\"location\": \"New York\", \"radius\": 500, \"type\": \"restaurants\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: type", "attempts": 1 }, { "row": "tc-201", "input": "USER: I need to set a reminder for my doctor's appointment. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and time of your appointment? USER: The appointment is on 2022-09-15 at 10:00 AM.", "expected": "{\"name\":\"create_reminder\",\"arguments\":{\"reminder_text\":\"Doctor's appointment\",\"reminder_date\":\"2022-09-15\",\"reminder_time\":\"10:00\"}}", "got": "{\"name\": \"create_reminder\", \"arguments\": {\"reminder_date\": \"2022-09-15\", \"reminder_text\": \"doctor's appointment\", \"reminder_time\": \"10:00:00\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: reminder_time", "attempts": 1 }, { "row": "tc-219", "input": "USER: I am looking for a vegan Italian recipe. Can you help me find one?", "expected": "{\"name\":\"search_recipes\",\"arguments\":{\"query\":\"vegan\",\"cuisine\":\"Italian\",\"diet\":\"vegan\"}}", "got": "{\"name\": \"search_recipes\", \"arguments\": {\"query\": \"Italian\", \"cuisine\": \"Italian\", \"diet\": \"vegan\"}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: query", "attempts": 1 } ], "deepseek/deepseek-v4.1-flash": [ { "row": "tc-26", "input": "USER: Hi, I need to convert 1000 USD to Euros. Can you help me with that?", "expected": "{\"name\":\"convert_currency\",\"arguments\":{\"amount\":1000,\"from\":\"USD\",\"to\":\"EUR\"}}", "got": "{\"name\": \"convert_currency\", \"arguments\": {\"amount\": 1000, \"from\": \"USD\", \"to\": \"Euros\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: to", "attempts": 1 }, { "row": "tc-64", "input": "USER: Hi, I need to convert 1000 USD to Euros. Can you help me with that? ASSISTANT: Sure, I can help you with that. Let me do the conversion for you.", "expected": "{\"name\":\"convert_currency\",\"arguments\":{\"amount\":1000,\"from_currency\":\"USD\",\"to_currency\":\"EUR\"}}", "got": "{\"name\": \"convert_currency\", \"arguments\": {\"amount\": 1000, \"from_currency\": \"USD\", \"to_currency\": \"Euros\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: to_currency", "attempts": 1 }, { "row": "tc-104", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee.", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a\",\"author\":\"Harper Lee\"}}", "got": "{\"name\": \"search_book\", \"arguments\": {\"title\": \"To Kill a...\", \"author\": \"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\": \"create_event\", \"arguments\": {\"title\": \"Birthday Party\", \"date\": \"2022-05-15\", \"location\": \"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_title\": \"Team Meeting\", \"start_time\": \"10:00 AM tomorrow\", \"end_time\": \"11:00 AM tomorrow\", \"location\": \"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-130", "input": "USER: I'm in the mood for Italian food. Can you find any Italian restaurants near me?", "expected": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\",\"location\":\"near me\"}}", "got": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: location", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\": \"translate_text\", \"arguments\": {\"text\": \"Je suis très heureux de vous rencontrer\", \"source_language\": \"French\", \"target_language\": \"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-178", "input": "USER: Hey, I need to calculate my carbon footprint for driving my car for 5 hours. Can you help me with that? ASSISTANT: Sure, I can help you with that. Let me calculate it for you.", "expected": "{\"name\":\"calculate_carbon_footprint\",\"arguments\":{\"activity\":\"driving\",\"duration\":5}}", "got": "{\"name\": \"calculate_carbon_footprint\", \"arguments\": {\"activity\": \"driving my car\", \"duration\": 5}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: activity", "attempts": 1 }, { "row": "tc-200", "input": "USER: I am currently in New York and I am looking for nearby restaurants within a 500 meter radius. Can you help me with that?", "expected": "{\"name\":\"find_nearby_places\",\"arguments\":{\"location\":\"New York\",\"radius\":500,\"type\":\"restaurant\"}}", "got": "{\"name\": \"find_nearby_places\", \"arguments\": {\"location\": \"New York\", \"radius\": 500, \"type\": \"restaurants\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: type", "attempts": 1 }, { "row": "tc-212", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, and end time for the meeting? USER: The meeting is titled \"Project Discussion\". It should start at 10:00 AM and end at 11:30 AM.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"title\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"title\": \"Project Discussion\", \"start_time\": \"tomorrow 10:00 AM\", \"end_time\": \"tomorrow 11:30 AM\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-219", "input": "USER: I am looking for a vegan Italian recipe. Can you help me find one?", "expected": "{\"name\":\"search_recipes\",\"arguments\":{\"query\":\"vegan\",\"cuisine\":\"Italian\",\"diet\":\"vegan\"}}", "got": "{\"name\": \"search_recipes\", \"arguments\": {\"query\": \"vegan Italian recipe\", \"cuisine\": \"Italian\", \"diet\": \"vegan\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: query", "attempts": 1 }, { "row": "tc-273", "input": "USER: Can you generate an invoice for my recent purchase? ASSISTANT: Of course! I would need some information to generate the invoice. Could you please provide the customer name, total amount, and details of the items purchased? USER: Sure, the customer name is John Doe. The total amount is $200. The items purchased ar …", "expected": "{\"name\":\"generate_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"total_amount\":200,\"items\":[{\"name\":\"book\",\"quantity\":2,\"price\":50},{\"name\":\"pen\",\"quantity\":4,\"price\":25}]}}", "got": "{\"name\": \"generate_invoice\", \"arguments\": {\"customer_name\": \"John Doe\", \"total_amount\": 200, \"items\": [{\"name\": \"books\", \"quantity\": 2, \"price\": 50}, {\"name\": \"pens\", \"quantity\": 4, \"price\": 25}]}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: items", "attempts": 1 } ], "qwen/qwen3.8-flash": [ { "row": "tc-10", "input": "USER: I want to watch a movie. Can you suggest me some? ASSISTANT: Of course! Could you please tell me your preferred genre and the year of release you're interested in? USER: I like action movies. I would prefer something released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2010}}", "got": "{\"name\": \"search_movies\", \"arguments\": {\"genre\": \"action\", \"year\": 2011}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: year", "attempts": 1 }, { "row": "tc-52", "input": "USER: Hi, I need help with calculating my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\": \"calculate_tax\", \"arguments\": {\"income\": 50000, \"tax_rate\": 0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 2 }, { "row": "tc-75", "input": "USER: I want to watch a movie tonight. Can you suggest me some good ones? ASSISTANT: Sure, I can help with that. Can you tell me your preferences? Like any specific genre, actor or year of release? USER: I like action movies. And I prefer movies released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"search_query\":\"action\",\"year\":2010}}", "got": "{\"name\": \"search_movies\", \"arguments\": {\"search_query\": \"action\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: year", "attempts": 2 }, { "row": "tc-104", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee.", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a\",\"author\":\"Harper Lee\"}}", "got": "{\"name\": \"search_book\", \"arguments\": {\"title\": \"To Kill a Mockingbird\", \"author\": \"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 2 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\": \"create_event\", \"arguments\": {\"title\": \"Birthday party\", \"date\": \"2022-05-15\", \"location\": \"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-120", "input": "USER: Hi, I had a cheeseburger for lunch today. It was about 563 calories. Can you log this for me? ASSISTANT: Sure, I can help with that. Let me log this information for you.", "expected": "{\"name\":\"track_calories\",\"arguments\":{\"meal\":\"cheeseburger\",\"calories\":563,\"date\":\"today\"}}", "got": "{\"name\": \"track_calories\", \"arguments\": {\"meal\": \"cheeseburger\", \"calories\": 563, \"date\": \"2026-06-16\"}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: date", "attempts": 1 }, { "row": "tc-122", "input": "USER: I need to schedule a meeting for tomorrow. ASSISTANT: Sure, I can help with that. Could you please provide me with the details of the meeting such as the name of the event, start time, end time, and location? USER: The meeting is called \"Project Discussion\". It will start at 10:00 AM and end at 11:30 AM. The loca …", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_name\":\"Project Discussion\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:30 AM\",\"location\":\"Conference Room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_name\": \"Project Discussion\", \"start_time\": \"tomorrow 10:00 AM\", \"end_time\": \"tomorrow 11:30 AM\", \"location\": \"Conference Room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_title\": \"Team Meeting\", \"start_time\": \"10:00 AM tomorrow\", \"end_time\": \"11:00 AM tomorrow\", \"location\": \"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-130", "input": "USER: I'm in the mood for Italian food. Can you find any Italian restaurants near me?", "expected": "{\"name\":\"search_restaurants\",\"arguments\":{\"cuisine\":\"Italian\",\"location\":\"near me\"}}", "got": "{\"name\": \"search_restaurants\", \"arguments\": {\"cuisine\": \"Italian\"}}", "outcome": "wrong", "kind": "missing_argument", "reason": "wrong: location", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\": \"translate_text\", \"arguments\": {\"text\": \"Je suis très heureux de vous rencontrer\", \"source_language\": \"French\", \"target_language\": \"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-171", "input": "USER: Hi, I need to calculate the interest on my savings. I have $5000 in my account and the interest rate is 2% per year. I plan to keep the money in the account for 3 years. Can you help me calculate the interest?", "expected": "{\"name\":\"calculate_interest\",\"arguments\":{\"principal\":5000,\"rate\":2,\"time\":3}}", "got": "{\"name\": \"calculate_interest\", \"arguments\": {\"principal\": 5000, \"rate\": 0.02, \"time\": 3}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: rate", "attempts": 1 }, { "row": "tc-177", "input": "USER: I need to create an invoice for a customer named John Doe. He bought 2 apples for $1 each and 3 bananas for $0.5 each. The tax rate is 10%.", "expected": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apple\",\"price\":1,\"quantity\":2},{\"name\":\"banana\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "got": "{\"name\": \"create_invoice\", \"arguments\": {\"customer_name\": \"John Doe\", \"items\": [{\"name\": \"apples\", \"price\": 1, \"quantity\": 2}, {\"name\": \"bananas\", \"price\": 0.5, \"quantity\": 3}], \"tax_rate\": 10}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: items", "attempts": 1 } ], "mistralai/mistral-small-2603": [ { "row": "tc-2", "input": "USER: Hi, I need a random number between 1 and 100.", "expected": "{\"name\":\"generate_random_number\",\"arguments\":{\"min\":1,\"max\":100}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-10", "input": "USER: I want to watch a movie. Can you suggest me some? ASSISTANT: Of course! Could you please tell me your preferred genre and the year of release you're interested in? USER: I like action movies. I would prefer something released after 2010.", "expected": "{\"name\":\"search_movies\",\"arguments\":{\"genre\":\"action\",\"year\":2010}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-15", "input": "USER: Can you get me the latest sports news from the United States?", "expected": "{\"name\":\"get_news\",\"arguments\":{\"category\":\"sports\",\"country\":\"United States\"}}", "got": "{\"name\": \"get_news\", \"arguments\": {\"category\": \"sports\", \"country\": \"US\"}}", "outcome": "wrong", "kind": "different_value", "reason": "wrong: country", "attempts": 1 }, { "row": "tc-19", "input": "USER: Hi, I need help with calculating a tip. My bill amount is $50 and I want to give a 20% tip.", "expected": "{\"name\":\"calculate_tip\",\"arguments\":{\"bill_amount\":50,\"tip_percentage\":20}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-21", "input": "USER: Hi, I need to calculate the area of a rectangle. The length is 10 units and the width is 5 units.", "expected": "{\"name\":\"calculate_rectangle_area\",\"arguments\":{\"length\":10,\"width\":5}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-37", "input": "USER: Hi, I need some help with a loan I'm considering. ASSISTANT: Of course, I'd be happy to assist you. Could you please provide me with the principal amount of the loan, the annual interest rate, and the loan term in years? USER: Sure, the principal amount is $200,000, the annual interest rate is 3.5%, and the loan …", "expected": "{\"name\":\"calculate_loan_payment\",\"arguments\":{\"principal\":200000,\"interest_rate\":3.5,\"loan_term\":30}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-40", "input": "USER: I am looking for a vegan Italian recipe with tomatoes and basil. Can you help me find one? ASSISTANT: Sure, I can help you with that. Let me find the best options for you.", "expected": "{\"name\":\"search_recipes\",\"arguments\":{\"keywords\":[\"tomatoes\",\"basil\"],\"cuisine\":\"Italian\",\"dietary_restrictions\":[\"vegan\"]}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-41", "input": "USER: Hi, I need some help with my mortgage calculations. ASSISTANT: Of course, I'd be happy to help with that. Could you please provide me with the loan amount, the annual interest rate, and the term of the loan in years? USER: Sure, the loan amount is $200,000, the annual interest rate is 3.5%, and the loan term is 3 …", "expected": "{\"name\":\"calculate_mortgage_payment\",\"arguments\":{\"loan_amount\":200000,\"interest_rate\":3.5,\"loan_term\":30}}", "got": "{\"name\": \"calculate_mortgage_payment\", \"arguments\": {\"loan_amount\": 200000, \"interest_rate\": 0.035, \"loan_term\": 30}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: interest_rate", "attempts": 1 }, { "row": "tc-47", "input": "USER: What does the word 'serendipity' mean?", "expected": "{\"name\":\"get_definition\",\"arguments\":{\"word\":\"serendipity\"}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-51", "input": "USER: I want to listen to some music. Can you suggest me some songs from the artist 'Coldplay'?", "expected": "{\"name\":\"search_music\",\"arguments\":{\"artist\":\"Coldplay\"}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 }, { "row": "tc-52", "input": "USER: Hi, I need help with calculating my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\": \"calculate_tax\", \"arguments\": {\"income\": 50000, \"tax_rate\": 0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-60", "input": "USER: Hi, I need to know the distance between New York and Los Angeles.", "expected": "{\"name\":\"calculate_distance\",\"arguments\":{\"location1\":\"New York\",\"location2\":\"Los Angeles\"}}", "got": "", "outcome": "invalid", "kind": "empty_reply", "reason": "empty output", "attempts": 3 } ], "meta-llama/llama-4-maverick": [ { "row": "tc-26", "input": "USER: Hi, I need to convert 1000 USD to Euros. Can you help me with that?", "expected": "{\"name\":\"convert_currency\",\"arguments\":{\"amount\":1000,\"from\":\"USD\",\"to\":\"EUR\"}}", "got": "{\"name\": \"convert_currency\", \"arguments\": {\"amount\": 1000, \"from\": \"USD\", \"to\": \"Euros\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: to", "attempts": 1 }, { "row": "tc-52", "input": "USER: Hi, I need help with calculating my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\": \"calculate_tax\", \"arguments\": {\"income\": 50000, \"tax_rate\": 0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-64", "input": "USER: Hi, I need to convert 1000 USD to Euros. Can you help me with that? ASSISTANT: Sure, I can help you with that. Let me do the conversion for you.", "expected": "{\"name\":\"convert_currency\",\"arguments\":{\"amount\":1000,\"from_currency\":\"USD\",\"to_currency\":\"EUR\"}}", "got": "{\"name\": \"convert_currency\", \"arguments\": {\"amount\": 1000, \"from_currency\": \"USD\", \"to_currency\": \"Euros\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: to_currency", "attempts": 1 }, { "row": "tc-80", "input": "USER: Hi, I need to convert 500 USD to Euros. Can you help me with that?", "expected": "{\"name\":\"convert_currency\",\"arguments\":{\"amount\":500,\"from_currency\":\"USD\",\"to_currency\":\"EUR\"}}", "got": "{\"name\": \"convert_currency\", \"arguments\": {\"amount\": 500, \"from_currency\": \"USD\", \"to_currency\": \"Euros\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: to_currency", "attempts": 1 }, { "row": "tc-97", "input": "USER: Hi, I need to convert 1000 USD to Euros. Can you help me with that? ASSISTANT: Sure, I can help with that. Let me do the conversion for you.", "expected": "{\"name\":\"convert_currency\",\"arguments\":{\"amount\":1000,\"from_currency\":\"USD\",\"to_currency\":\"EUR\"}}", "got": "{\"name\": \"convert_currency\", \"arguments\": {\"amount\": 1000, \"from_currency\": \"USD\", \"to_currency\": \"Euros\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: to_currency", "attempts": 1 }, { "row": "tc-103", "input": "USER: Hi, I need to calculate my tax. My income is $50000 and the tax rate is 20%.", "expected": "{\"name\":\"calculate_tax\",\"arguments\":{\"income\":50000,\"tax_rate\":20}}", "got": "{\"name\": \"calculate_tax\", \"arguments\": {\"income\": 50000, \"tax_rate\": 0.2}}", "outcome": "wrong", "kind": "number_format", "reason": "wrong: tax_rate", "attempts": 1 }, { "row": "tc-110", "input": "USER: I need to create an event for my birthday party. ASSISTANT: Sure, I can help with that. Could you please provide me with the date and location for your birthday party? USER: The date is 2022-05-15 and the location is my house.", "expected": "{\"name\":\"create_event\",\"arguments\":{\"title\":\"My Birthday Party\",\"date\":\"2022-05-15\",\"location\":\"My House\"}}", "got": "{\"name\": \"create_event\", \"arguments\": {\"title\": \"birthday party\", \"date\": \"2022-05-15\", \"location\": \"my house\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-119", "input": "USER: I am looking for a book but I can't remember the full title. It's something like \"To Kill a...\" and it's by Harper Lee. Can you help me find it?", "expected": "{\"name\":\"search_book\",\"arguments\":{\"title\":\"To Kill a...\",\"author\":\"Harper Lee\"}}", "got": "{\"name\": \"search_book\", \"arguments\": {\"title\": \"To Kill a\", \"author\": \"Harper Lee\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: title", "attempts": 1 }, { "row": "tc-128", "input": "USER: I need to create a new event in my calendar. ASSISTANT: Sure, I can help with that. Could you please provide me with the title, start time, end time, and location of the event? USER: The title is \"Team Meeting\", it will start at 10:00 AM and end at 11:00 AM tomorrow. The location is the conference room.", "expected": "{\"name\":\"create_calendar_event\",\"arguments\":{\"event_title\":\"Team Meeting\",\"start_time\":\"10:00 AM\",\"end_time\":\"11:00 AM\",\"location\":\"conference room\"}}", "got": "{\"name\": \"create_calendar_event\", \"arguments\": {\"event_title\": \"Team Meeting\", \"start_time\": \"10:00 AM tomorrow\", \"end_time\": \"11:00 AM tomorrow\", \"location\": \"conference room\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: start_time, end_time", "attempts": 1 }, { "row": "tc-167", "input": "USER: Hi, I have a sentence in French that I need translated to English. The sentence is \"Je suis très heureux de vous rencontrer\".", "expected": "{\"name\":\"translate_text\",\"arguments\":{\"text\":\"Je suis très heureux de vous rencontrer\",\"source_language\":\"fr\",\"target_language\":\"en\"}}", "got": "{\"name\": \"translate_text\", \"arguments\": {\"text\": \"Je suis tr\\u00e8s heureux de vous rencontrer\", \"source_language\": \"French\", \"target_language\": \"English\"}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: source_language, target_language", "attempts": 1 }, { "row": "tc-174", "input": "USER: I need to create a new todo list for my weekend. ASSISTANT: Sure, I can help with that. What would you like to title your todo list? USER: Let's call it \"Weekend Chores\". ASSISTANT: Great! What tasks would you like to add to your \"Weekend Chores\" list? USER: I need to do laundry, grocery shopping, and clean the h …", "expected": "{\"name\":\"create_todo_list\",\"arguments\":{\"title\":\"Weekend Chores\",\"tasks\":[\"do laundry\",\"grocery shopping\",\"clean the house\"]}}", "got": "{\"name\": \"create_todo_list\", \"arguments\": {\"title\": \"Weekend Chores\", \"tasks\": [\"laundry\", \"grocery shopping\", \"clean the house\"]}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: tasks", "attempts": 1 }, { "row": "tc-177", "input": "USER: I need to create an invoice for a customer named John Doe. He bought 2 apples for $1 each and 3 bananas for $0.5 each. The tax rate is 10%.", "expected": "{\"name\":\"create_invoice\",\"arguments\":{\"customer_name\":\"John Doe\",\"items\":[{\"name\":\"apple\",\"price\":1,\"quantity\":2},{\"name\":\"banana\",\"price\":0.5,\"quantity\":3}],\"tax_rate\":10}}", "got": "{\"name\": \"create_invoice\", \"arguments\": {\"customer_name\": \"John Doe\", \"items\": [{\"name\": \"apples\", \"price\": 1, \"quantity\": 2}, {\"name\": \"bananas\", \"price\": 0.5, \"quantity\": 3}], \"tax_rate\": 10}}", "outcome": "wrong", "kind": "wording", "reason": "wrong: items", "attempts": 1 } ] }, "failed_rows": { "openai/gpt-6.1-sol": [ "tc-52", "tc-75", "tc-104", "tc-103", "tc-110", "tc-120", "tc-122", "tc-128", "tc-130", "tc-167", "tc-177", "tc-178", "tc-200", "tc-201", "tc-212", "tc-219", "tc-226", "tc-273" ], "openai/gpt-6.1-terra": [ "tc-10", "tc-52", "tc-75", "tc-103", "tc-104", "tc-110", "tc-120", "tc-122", "tc-128", "tc-130", "tc-167", "tc-171", "tc-177", "tc-178", "tc-200", "tc-212", "tc-219", "tc-226", "tc-273" ], "openai/gpt-6-luna": [ "tc-10", "tc-43", "tc-52", "tc-75", "tc-104", "tc-110", "tc-119", "tc-120", "tc-122", "tc-128", "tc-162", "tc-167", "tc-177", "tc-178", "tc-200", "tc-201", "tc-212", "tc-219", "tc-226", "tc-248", "tc-273" ], "anthropic/claude-haiku-5.5": [ "tc-10", "tc-75", "tc-110", "tc-119", "tc-120", "tc-122", "tc-128", "tc-130", "tc-167", "tc-177", "tc-178", "tc-212", "tc-219", "tc-226", "tc-248", "tc-262" ], "google/gemini-3.5-flash-lite": [ "tc-75", "tc-103", "tc-110", "tc-119", "tc-128", "tc-130", "tc-167", "tc-171", "tc-177", "tc-200", "tc-201", "tc-219", "tc-226", "tc-262", "tc-273" ], "deepseek/deepseek-v4.1-flash": [ "tc-26", "tc-64", "tc-104", "tc-110", "tc-128", "tc-130", "tc-167", "tc-178", "tc-200", "tc-212", "tc-219", "tc-273" ], "qwen/qwen3.8-flash": [ "tc-10", "tc-52", "tc-75", "tc-104", "tc-110", "tc-120", "tc-122", "tc-128", "tc-130", "tc-167", "tc-171", "tc-177", "tc-178", "tc-200", "tc-203", "tc-201", "tc-212", "tc-219", "tc-226", "tc-248", "tc-273" ], "mistralai/mistral-small-2603": [ "tc-2", "tc-10", "tc-15", "tc-19", "tc-21", "tc-37", "tc-40", "tc-41", "tc-47", "tc-51", "tc-52", "tc-60", "tc-66", "tc-79", "tc-82", "tc-83", "tc-87", "tc-92", "tc-94", "tc-102", "tc-103", "tc-106", "tc-110", "tc-114", "tc-116", "tc-119", "tc-120", "tc-121", "tc-122", "tc-123", "tc-124", "tc-126", "tc-128", "tc-130", "tc-131", "tc-138", "tc-140", "tc-141", "tc-142", "tc-145", "tc-151", "tc-160", "tc-161", "tc-162", "tc-164", "tc-171", "tc-177", "tc-178", "tc-180", "tc-181", "tc-186", "tc-189", "tc-191", "tc-192", "tc-195", "tc-197", "tc-199", "tc-200", "tc-201", "tc-202", "tc-203", "tc-207", "tc-210", "tc-212", "tc-215", "tc-216", "tc-218", "tc-219", "tc-225", "tc-226", "tc-232", "tc-242", "tc-243", "tc-245", "tc-247", "tc-251", "tc-258", "tc-260", "tc-261", "tc-271", "tc-272", "tc-276", "tc-279", "tc-284", "tc-292", "tc-293", "tc-295", "tc-296" ], "meta-llama/llama-4-maverick": [ "tc-26", "tc-52", "tc-64", "tc-80", "tc-97", "tc-103", "tc-110", "tc-119", "tc-128", "tc-167", "tc-174", "tc-177", "tc-178", "tc-212", "tc-219", "tc-273" ] }, "run_cost_usd": 0.503184, "spend_cap_usd": 1.2, "capped": false, "cost_note": "cost = sum of OpenRouter usage.cost over every attempt, retries included" }