[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-ai-can-fake-early-ux-testing-but-watch-for-fabricated-feedback":10,"sections":34},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":24,"tags":25,"sources":29,"feedback":33,"feedback_at":22,"cost_usd":33,"total_tokens":33},5045,"ai-can-fake-early-ux-testing-but-watch-for-fabricated-feedback","AI Can Fake Early UX Testing, but Watch for Fabricated Feedback","A new arXiv paper tests AI-simulated user feedback against real reviews, finding it promising but still prone to instability and fabrication.","A new study asks a blunt question: can AI-generated personas stand in for the users your startup can't afford to test with?\n\nResearchers built a pipeline that has large language models play the role of test users - walking through a product, flagging friction points, and answering micro-survey questions - all driven by versioned prompts, personas, and UI snapshots. Because no dataset of real usability studies exists to check the work against, the team validated the simulated feedback against six proxy corpora: app store reviews, support tweets, and open-source issue trackers. They compared three ways of scoring how closely the AI feedback matched real complaints - keyword matching, TF-IDF, and multilingual embeddings - and found embeddings performed far better, scoring 0.128 versus 0.000 for keyword matching on the Gojek app-review dataset. They also found that a popular shortcut metric, top-k Jaccard overlap, makes the simulation look more accurate than it actually is once k gets large.\n\nFor teams without budget for user research, an AI stand-in for testers is an obvious pitch - cheaper and faster than recruiting real people. But the paper's own numbers undercut that pitch: bootstrap resampling showed the alignment scores swing wildly, and an adversarial judge model caught the system fabricating feedback outright in some cases. That is the real finding here - not that AI can simulate users, but that measuring whether it did so honestly is still unsolved.\n\nTreat this the way you'd treat a focus group run by a chatbot: useful for catching obvious problems early, no substitute for asking actual humans before you ship.","[\"ai\",\"ux research\",\"llm agents\",\"startups\"]","2026-08-17T04:00:00.000Z","2026-08-17T07:09:10.457Z","2026-08-17T07:09:22.268Z","published",null,[],"ai",[24,26,27,28],"ux research","llm agents","startups",[30],{"name":31,"url":32},"arXiv cs.AI","https:\u002F\u002Farxiv.org\u002Fabs\u002F2608.13563",0,{"sections":35},[36,40,44,49,54,59,64,69,74,79,83,88,93,98],{"name":37,"slug":24,"count":38,"latest_published_at":39},"AI",3293,"2026-08-20T04:00:00.000Z",{"name":41,"slug":42,"count":43,"latest_published_at":39},"Security","security",435,{"name":45,"slug":46,"count":47,"latest_published_at":48},"Policy","policy",210,"2026-08-19T09:32:27.000Z",{"name":50,"slug":51,"count":52,"latest_published_at":53},"Deals","deals",179,"2026-06-29T20:02:07.000Z",{"name":55,"slug":56,"count":57,"latest_published_at":58},"Hardware","hardware",140,"2026-08-19T18:25:42.000Z",{"name":60,"slug":61,"count":62,"latest_published_at":63},"Consumer Tech","consumer-tech",95,"2026-08-18T16:05:00.000Z",{"name":65,"slug":66,"count":67,"latest_published_at":68},"Science","science",90,"2026-08-19T18:41:02.000Z",{"name":70,"slug":71,"count":72,"latest_published_at":73},"Software","software",73,"2026-08-18T07:51:50.000Z",{"name":75,"slug":76,"count":77,"latest_published_at":78},"Dev Tools","dev-tools",69,"2026-08-18T04:00:00.000Z",{"name":80,"slug":28,"count":81,"latest_published_at":82},"Startups",47,"2026-08-19T19:13:46.000Z",{"name":84,"slug":85,"count":86,"latest_published_at":87},"Gaming","gaming",41,"2026-07-09T04:00:00.000Z",{"name":89,"slug":90,"count":91,"latest_published_at":92},"General","general",33,"2026-08-18T22:18:13.000Z",{"name":94,"slug":95,"count":96,"latest_published_at":97},"Reviews","reviews",20,"2026-06-24T12:00:01.000Z",{"name":99,"slug":100,"count":101,"latest_published_at":102},"How-To","how-to",6,"2026-06-16T09:00:00.000Z"]