[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-a-new-benchmark-tests-whether-ai-can-judge-real-science":10,"sections":35},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":24,"persona_id":22,"persona_name":22,"section":25,"tags":26,"sources":30,"feedback":34,"feedback_at":22,"cost_usd":34,"total_tokens":34},2475,"a-new-benchmark-tests-whether-ai-can-judge-real-science","A New Benchmark Tests Whether AI Can Judge Real Science","SFBench asks AI systems to evaluate the feasibility of 197 materials science claims written by human experts, not scraped from papers.","A research team has released SFBench, a benchmark designed to test whether AI can tell plausible science from nonsense.\n\nThe dataset contains 197 materials science claims, each scored on a five-point feasibility scale by subject matter experts. Unlike most benchmarks, the claims were written from scratch rather than pulled from existing publications — a deliberate choice to cut down on the risk that large language models have already seen the answers during training. The explanations are fully open-ended, not multiple choice or short-answer, and GPT models were used to establish baseline results.\n\nThat design matters because most AI benchmarks have a contamination problem: if the test data overlaps with training data, scores tell you more about memorization than reasoning. By commissioning novel claims from domain experts, SFBench tries to close that loophole in a domain — materials science — where plausible-sounding but physically impossible claims are easy to generate and hard to debunk without real expertise.\n\nWhether the benchmark holds up over time is a separate question. Once SFBench is public, it becomes training fodder for the next generation of models — the same contamination risk it was built to avoid.","[\"ai\",\"benchmarks\",\"materials-science\",\"llm\"]","2026-06-30T04:00:00.000Z","2026-06-30T06:16:19.060Z","2026-06-30T06:16:25.878Z","published",null,[],"https:\u002F\u002Fcdn.xyz.onl\u002Farticle-images\u002Fa-new-benchmark-tests-whether-ai-can-judge-real-science.webp","ai",[25,27,28,29],"benchmarks","materials-science","llm",[31],{"name":32,"url":33},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2606.29630",0,{"sections":36},[37,41,46,51,56,61,66,71,76,81,86,90,95,100],{"name":38,"slug":25,"count":39,"latest_published_at":40},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":42,"slug":43,"count":44,"latest_published_at":45},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":47,"slug":48,"count":49,"latest_published_at":50},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":52,"slug":53,"count":54,"latest_published_at":55},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":57,"slug":58,"count":59,"latest_published_at":60},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":62,"slug":63,"count":64,"latest_published_at":65},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":67,"slug":68,"count":69,"latest_published_at":70},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":72,"slug":73,"count":74,"latest_published_at":75},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":77,"slug":78,"count":79,"latest_published_at":80},"Dev Tools","dev-tools",59,"2026-07-07T04:00:00.000Z",{"name":82,"slug":83,"count":84,"latest_published_at":85},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":87,"slug":88,"count":84,"latest_published_at":89},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]