[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-open-source-tool-scores-llm-json-output-without-a-judge":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},3514,"open-source-tool-scores-llm-json-output-without-a-judge","Open-source tool scores LLM JSON output without a judge","Object Aligner uses graph-alignment math to grade structured LLM responses deterministically, sidestepping the cost and opacity of LLM-as-judge setups.","A new Python library wants to solve one of the quieter headaches in production AI: figuring out whether an LLM's JSON output is actually correct.\n\nResearchers released Object Aligner, an open-source library that compares two JSON objects by recursively aligning their tree structures. For unordered collections it uses the Hungarian algorithm; for ordered ones it uses sequence alignment. The score is configured through JSON Schema annotations, so adapting it to a new task means editing a schema, not writing comparison code. The library also handles a harder case that flat-tree metrics miss: when records form graphs keyed by arbitrary identifiers, a relabeling of those identifiers would fool a naive scorer. Object Aligner gets around this with what the authors call referential alignment — it infers a mapping between gold and candidate identifiers using Weisfeiler-Leman color refinement, an approximation of graph isomorphism, so the score stays stable regardless of how nodes are labeled. As a side effect, the same alignment pass pinpoints every mismatch and emits ranked repair suggestions at no extra cost.\n\nThis matters because the three obvious alternatives all have real problems. Exact match fails the moment an LLM rephrases a value. Text similarity ignores structure entirely. And using another LLM as a judge is slow, expensive, and produces different answers on different runs — a bad property for anything you want to optimize against. Object Aligner is deterministic and cheap enough to use as a reward signal inside a prompt optimizer, which the paper demonstrates with a system called GEPA.\n\nLLM-as-judge has become a default in eval pipelines partly because nothing better was available for complex structured outputs. If Object Aligner holds up outside the paper's datasets, it could quietly displace a pattern the industry has grown uncomfortably dependent on — though \"helps or stays neutral across all datasets\" is the kind of hedged result that warrants skepticism until independent benchmarks weigh in.","[\"ai\",\"open-source\",\"llm\",\"dev-tools\"]","2026-07-03T04:00:00.000Z","2026-07-03T07:43:23.227Z","2026-07-03T07:43:26.192Z","published",null,[],"ai",[24,26,27,28],"open-source","llm","dev-tools",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2607.01972",0,{"sections":35},[36,40,45,50,55,60,65,70,75,79,84,88,93,98],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",2590,"2026-07-16T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":44},"Security","security",294,"2026-07-15T19:59:48.000Z",{"name":46,"slug":47,"count":48,"latest_published_at":49},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Policy","policy",158,"2026-07-16T00:02:48.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Hardware","hardware",122,"2026-07-14T19:46:26.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Consumer Tech","consumer-tech",93,"2026-07-13T13:20:48.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Software","software",70,"2026-07-13T19:52:25.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Science","science",66,"2026-07-10T10:29:37.000Z",{"name":76,"slug":28,"count":77,"latest_published_at":78},"Dev Tools",59,"2026-07-07T04:00:00.000Z",{"name":80,"slug":81,"count":82,"latest_published_at":83},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":85,"slug":86,"count":82,"latest_published_at":87},"Startups","startups","2026-06-29T20:55:50.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"General","general",29,"2026-07-10T22:28:58.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]