[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ask-seven-ai-models-to-score-a-call-get-seven-answers":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},8202,"ask-seven-ai-models-to-score-a-call-get-seven-answers","Ask Seven AI Models to Score a Call, Get Seven Answers","Seven AI models graded the same earnings calls and produced wildly different scores, undermining a common LLM research shortcut.","Ask seven different AI models to grade the same earnings call, and you'll get seven different report cards.\n\nA new study tested thirteen LLM-based measures - things like sentiment, management clarity, uncertainty, and climate or political risk - using seven large language models from different providers. Each model scored the same set of S&P 500 earnings call transcripts on these constructs. The models agreed with each other only about half the time: cross-model rank correlations averaged 0.52. Differences that showed up consistently across all seven providers, the kind you'd expect if the underlying text really were ambiguous, explained just 34% of the variation in scores.\n\nThat gap matters because researchers increasingly treat these AI-generated scores as objective facts about a company, then feed them into models that predict analyst or market behavior. The study found that model disagreement does not track with real disagreement among analysts or investors, which suggests the variation is mostly an artifact of which model you picked, not a signal about the company. Swap one model for another in a research pipeline, and coefficients, significance levels, even the sign of a result can flip.\n\nAveraging scores across providers smooths out the rankings, but the number you ultimately report still depends on which models went into that average - so \"the AI said so\" is only as reliable as the answer to \"which AI, though?\"","[\"ai\",\"llms\",\"research\",\"finance\"]","2026-09-28T04:00:00.000Z","2026-09-28T16:41:28.424Z","2026-09-28T16:41:35.658Z","published",null,[],"ai",[24,26,27,28],"llms","research","finance",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2609.31013",0,{"sections":35},[36,39,43,48,53,58,62,67,72,77,82,87,91,96],{"name":37,"slug":24,"count":38,"latest_published_at":18},"AI",4844,{"name":40,"slug":41,"count":42,"latest_published_at":18},"Security","security",762,{"name":44,"slug":45,"count":46,"latest_published_at":47},"Policy","policy",399,"2026-09-27T18:39:02.000Z",{"name":49,"slug":50,"count":51,"latest_published_at":52},"Deals","deals",266,"2026-09-28T14:00:00.000Z",{"name":54,"slug":55,"count":56,"latest_published_at":57},"Hardware","hardware",189,"2026-09-28T10:52:40.000Z",{"name":59,"slug":60,"count":61,"latest_published_at":18},"Science","science",151,{"name":63,"slug":64,"count":65,"latest_published_at":66},"Consumer Tech","consumer-tech",135,"2026-09-26T14:30:00.000Z",{"name":68,"slug":69,"count":70,"latest_published_at":71},"Software","software",91,"2026-09-25T20:55:00.000Z",{"name":73,"slug":74,"count":75,"latest_published_at":76},"Dev Tools","dev-tools",84,"2026-09-26T04:20:58.000Z",{"name":78,"slug":79,"count":80,"latest_published_at":81},"Startups","startups",76,"2026-09-25T18:33:59.000Z",{"name":83,"slug":84,"count":85,"latest_published_at":86},"Gaming","gaming",48,"2026-09-25T18:35:21.000Z",{"name":88,"slug":89,"count":85,"latest_published_at":90},"General","general","2026-09-26T17:02:42.000Z",{"name":92,"slug":93,"count":94,"latest_published_at":95},"Reviews","reviews",30,"2026-09-24T20:07:31.000Z",{"name":97,"slug":98,"count":99,"latest_published_at":100},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]