{
  "url": "https://www.nytimes.com/section/world",
  "userAgent": "GPTBot",
  "path": "/section/world",
  "robotsTxt": {
    "url": "https://www.nytimes.com/robots.txt",
    "status": 200,
    "found": true
  },
  "allowed": false,
  "rule": "Disallow: /",
  "group": [
    "gptbot"
  ],
  "crawlDelay": null,
  "sitemaps": [
    "https://www.nytimes.com/sitemaps/new/news.xml.gz",
    "https://www.nytimes.com/sitemaps/new/sitemap.xml.gz",
    "https://www.nytimes.com/sitemaps/new/collections.xml.gz",
    "https://www.nytimes.com/sitemaps/new/video.xml.gz",
    "https://www.nytimes.com/sitemaps/new/cooking.xml.gz",
    "https://www.nytimes.com/sitemaps/new/recipe-collects.xml.gz",
    "https://www.nytimes.com/sitemaps/new/regions.xml",
    "https://www.nytimes.com/sitemaps/new/best-sellers.xml",
    "https://www.nytimes.com/sitemaps/new/subscription-landing-pages.xml",
    "https://www.nytimes.com/sitemaps/new/weather.xml.gz"
  ],
  "llmsTxt": false,
  "aiCrawlers": {
    "GPTBot": false,
    "ChatGPT-User": false,
    "OAI-SearchBot": false,
    "ClaudeBot": false,
    "Claude-User": false,
    "anthropic-ai": false,
    "PerplexityBot": false,
    "Google-Extended": false,
    "CCBot": false,
    "Bytespider": false,
    "Applebot-Extended": false,
    "meta-externalagent": false
  },
  "note": "RFC 9309 evaluation (longest match wins, Allow wins ties). robots.txt is a crawler convention, not a license; check the site's terms too.",
  "freeTier": {
    "used": 1,
    "limit": 10,
    "remainingToday": 9
  },
  "payment": {
    "charged": false
  }
}