[{"data":1,"prerenderedAt":-1},["ShallowReactive",2],{"branding":3,"analytics":7,"article-openai-will-honor-a-shared-robotstxt-signal-for-ai-training":10,"sections":44},{"siteName":4,"siteTagline":5,"publisherName":4,"contactEmail":6},"The Revision","Tech news, decoded.","editor@therevision.news",{"gaMeasurementId":8,"adsenseClientId":9},"G-ZW2MV82GYR","ca-pub-8533917693782264",{"article":11},{"id":12,"slug":13,"title":14,"dek":15,"body_md":16,"tags_json":17,"published_at":18,"created_at":19,"updated_at":20,"status":21,"review_note":22,"review_notes":23,"image_url":22,"persona_id":22,"persona_name":22,"section":30,"tags":31,"sources":36,"feedback":43,"feedback_at":22,"cost_usd":43,"total_tokens":43},12494,"openai-will-honor-a-shared-robotstxt-signal-for-ai-training","OpenAI Will Honor a Shared robots.txt Signal for AI Training","OpenAI says it will respect a common robots.txt signal that lets websites state their AI training preferences to participating providers.","OpenAI will honor a shared robots.txt signal that lets website operators tell participating AI providers how they want their content treated for training.\n\nThat is the whole announcement: one signal, carried in a file sites already publish, meant for providers that agree to recognize it. The company said the goal is to help operators communicate their preferences.\n\nSome background, which is context and not part of the announcement. Robots.txt is a plain text file, dating to the 1990s, that sits at the root of a site and tells automated crawlers which pages to skip. AI opt-outs have mostly worked company by company. A site names each crawler it wants to block, then has to keep that list current as new ones appear. A common signal would let an operator state a preference once.\n\nThe word \"participating\" carries a lot of weight. A shared signal is only as useful as the number of providers that agree to read it, and robots.txt has always relied on voluntary compliance rather than enforcement. Which other companies join, and how finely the signal can express a preference, will decide whether this simplifies anything for publishers or just adds one more entry to the list they already maintain.","[\"openai\",\"robots-txt\",\"ai-training\",\"web-crawlers\"]","2026-10-30T09:30:00.000Z","2026-11-08T05:04:44.169Z","2026-11-08T05:04:46.762Z","published",null,[24],{"id":25,"reviewer":26,"round":27,"reason":28,"status":29},"editor-r1","editor",1,"Remove the line 'It did not say, in the material available...' because it admits missing source material instead of dropping or sourcing the claims, and remove or verify the 'October 30' date, which does not appear in the source; then tighten the piece so it only states what OpenAI actually announced (it will honor a shared robots.txt signal for AI training preferences) plus clearly labeled context.","resolved","ai",[32,33,34,35],"openai","robots-txt","ai-training","web-crawlers",[37,40],{"name":38,"url":39},"OpenAI","https:\u002F\u002Fopenai.com\u002Findex\u002Fhelping-websites-express-ai-training-preferences",{"name":41,"url":42},"TechRadar","https:\u002F\u002Fwww.techradar.com\u002Fpro\u002Fgoogle-and-microsoft-among-those-signing-up-to-openais-plan-for-a-new-website-crawling-standard-including-opting-out-of-internet-scraping",0,{"sections":45},[46,50,55,60,65,70,75,80,85,90,95,100,105,110],{"name":47,"slug":30,"count":48,"latest_published_at":49},"AI",7110,"2026-11-08T14:34:09.000Z",{"name":51,"slug":52,"count":53,"latest_published_at":54},"Security","security",1099,"2026-11-08T15:24:01.000Z",{"name":56,"slug":57,"count":58,"latest_published_at":59},"Deals","deals",620,"2026-11-08T08:00:00.000Z",{"name":61,"slug":62,"count":63,"latest_published_at":64},"Policy","policy",604,"2026-11-07T15:48:26.000Z",{"name":66,"slug":67,"count":68,"latest_published_at":69},"Consumer Tech","consumer-tech",336,"2026-11-08T21:00:00.000Z",{"name":71,"slug":72,"count":73,"latest_published_at":74},"Hardware","hardware",316,"2026-11-08T14:36:39.000Z",{"name":76,"slug":77,"count":78,"latest_published_at":79},"Science","science",222,"2026-11-07T16:19:17.000Z",{"name":81,"slug":82,"count":83,"latest_published_at":84},"Software","software",154,"2026-11-08T15:42:52.000Z",{"name":86,"slug":87,"count":88,"latest_published_at":89},"Startups","startups",150,"2026-11-06T16:50:01.000Z",{"name":91,"slug":92,"count":93,"latest_published_at":94},"Dev Tools","dev-tools",125,"2026-11-06T22:15:59.000Z",{"name":96,"slug":97,"count":98,"latest_published_at":99},"General","general",111,"2026-11-08T21:27:29.000Z",{"name":101,"slug":102,"count":103,"latest_published_at":104},"Gaming","gaming",94,"2026-11-08T20:52:05.000Z",{"name":106,"slug":107,"count":108,"latest_published_at":109},"Reviews","reviews",38,"2026-10-20T12:00:00.000Z",{"name":111,"slug":112,"count":113,"latest_published_at":114},"How-To","how-to",8,"2026-10-05T09:00:00.000Z"]