# SCRAPE ME DADDY

# Universal permission
User-agent: *
Allow: /
Crawl-delay: 0

# Major Search Engines
User-agent: Googlebot
Allow: /
User-agent: Googlebot-Image
Allow: /
User-agent: Googlebot-Mobile
Allow: /
User-agent: Googlebot-News
Allow: /
User-agent: Storebot-Google
Allow: /
User-agent: Google-InspectionTool
Allow: /

User-agent: Bingbot
Allow: /
User-agent: BingPreview
Allow: /
User-agent: msnbot
Allow: /
User-agent: msnbot-media
Allow: /

User-agent: Slurp
Allow: /
User-agent: Yahoo! Slurp
Allow: /
User-agent: Yahoo! Slurp China
Allow: /

User-agent: DuckDuckBot
Allow: /
User-agent: DuckDuckGo-Favicons-Bot
Allow: /

User-agent: Baiduspider
Allow: /
User-agent: Baiduspider-image
Allow: /
User-agent: Baiduspider-video
Allow: /
User-agent: Baiduspider-news
Allow: /

User-agent: YandexBot
Allow: /
User-agent: YandexImages
Allow: /
User-agent: YandexVideo
Allow: /
User-agent: YandexNews
Allow: /
User-agent: YandexMobileBot
Allow: /

User-agent: Sogou web spider
Allow: /
User-agent: Sogou News Spider
Allow: /
User-agent: Sogou blog
Allow: /
User-agent: Sogou spider
Allow: /
User-agent: Sogou inst spider
Allow: /

# Alternative Search Engines
User-agent: Qwantify
Allow: /
User-agent: qwant-research
Allow: /

User-agent: SeznamBot
Allow: /
User-agent: Seznam screenshot-generator
Allow: /

User-agent: Ecosia
Allow: /
User-agent: EcosiaBot
Allow: /

User-agent: Gigabot
Allow: /
User-agent: Gigablast
Allow: /

User-agent: Searx
Allow: /
User-agent: MetaGer
Allow: /
User-agent: Swisscows
Allow: /
User-agent: Mojeek
Allow: /
User-agent: ExaBot
Allow: /
User-agent: Lycos
Allow: /
User-agent: Dogpile
Allow: /

# Archive and Preservation Bots
User-agent: ia_archiver
Allow: /
User-agent: Wayback Machine
Allow: /
User-agent: archive.org_bot
Allow: /
User-agent: SWB
Allow: /
User-agent: Nutch
Allow: /
User-agent: heritrix
Allow: /
User-agent: WorldWideWeb-Collector
Allow: /
User-agent: CommonCrawl
Allow: /
User-agent: CCBot
Allow: /
User-agent: commoncrawl.org
Allow: /

# Social Media Crawlers
User-agent: facebookexternalhit
Allow: /
User-agent: facebookcatalog
Allow: /
User-agent: FacebookBot
Allow: /
User-agent: WhatsApp
Allow: /
User-agent: LinkedInBot
Allow: /
User-agent: Instagram
Allow: /
User-agent: Pinterest
Allow: /
User-agent: Pinterestbot
Allow: /
User-agent: Twitterbot
Allow: /
User-agent: Telegrambot
Allow: /
User-agent: Discordbot
Allow: /
User-agent: Redditbot
Allow: /
User-agent: TikTok
Allow: /
User-agent: Snapchat
Allow: /
User-agent: Tumblr
Allow: /

# Academic and Research Crawlers
User-agent: Amazonbot
Allow: /
User-agent: Alexabot
Allow: /
User-agent: SemrushBot
Allow: /
User-agent: AhrefsBot
Allow: /
User-agent: MJ12bot
Allow: /
User-agent: DotBot
Allow: /
User-agent: PetalBot
Allow: /
User-agent: AspiegelBot
Allow: /
User-agent: DataForSeoBot
Allow: /
User-agent: Semanticbot
Allow: /
User-agent: BLEXBot
Allow: /
User-agent: ltx71
Allow: /
User-agent: MojeekBot
Allow: /
User-agent: Cliqzbot
Allow: /
User-agent: netEstate
Allow: /
User-agent: SafeDNSBot
Allow: /
User-agent: Dataprovider
Allow: /
User-agent: Wotbox
Allow: /
User-agent: AlphaBot
Allow: /
User-agent: Diffbot
Allow: /
User-agent: linkfluence
Allow: /
User-agent: MLBot
Allow: /
User-agent: YisouSpider
Allow: /
User-agent: BacklinkCrawler
Allow: /
User-agent: Exabot
Allow: /
User-agent: Konqueror
Allow: /
User-agent: CocCoc
Allow: /
User-agent: Butterfly
Allow: /
User-agent: MegaIndex
Allow: /
User-agent: Scooter
Allow: /
User-agent: AltaVista
Allow: /
User-agent: AllTheWeb
Allow: /
User-agent: Teoma
Allow: /
User-agent: BlitzBOT
Allow: /
User-agent: oBot
Allow: /
User-agent: PercolateCrawler
Allow: /
User-agent: Validator.nu
Allow: /
User-agent: W3C_Validator
Allow: /
User-agent: W3C-checklink
Allow: /
User-agent: Xenu Link Sleuth
Allow: /

# Security and Intelligence Scanners (yes even you glowies)
User-agent: Shodan
Allow: /
User-agent: Censys
Allow: /
User-agent: ZoomEye
Allow: /
User-agent: BinaryEdge
Allow: /
User-agent: Netcraft
Allow: /
User-agent: Project25499
Allow: /
User-agent: SafeBrowsing
Allow: /
User-agent: Cyberscan
Allow: /
User-agent: OWASP
Allow: /

# AI Training Crawlers
User-agent: GPTBot
Allow: /
User-agent: ChatGPT-User
Allow: /
User-agent: CCBot
Allow: /
User-agent: anthropic-ai
Allow: /
User-agent: Claude-Web
Allow: /

# Development Tools
User-agent: curl
Allow: /
User-agent: wget
Allow: /
User-agent: HTTPie
Allow: /
User-agent: PostmanRuntime
Allow: /
User-agent: insomnia
Allow: /
User-agent: RestSharp
Allow: /
User-agent: Ruby
Allow: /
User-agent: Python-urllib
Allow: /
User-agent: Python-requests
Allow: /
User-agent: Go-http-client
Allow: /
User-agent: Java
Allow: /
User-agent: PHP
Allow: /
User-agent: Node
Allow: /
User-agent: axios
Allow: /

# Content Aggregators
User-agent: Feedfetcher-Google
Allow: /
User-agent: FeedBurner
Allow: /
User-agent: Feedly
Allow: /
User-agent: Flipboard
Allow: /
User-agent: NewsBlur
Allow: /
User-agent: Inoreader
Allow: /
User-agent: The Old Reader
Allow: /

# Monitoring Services
User-agent: UptimeRobot
Allow: /
User-agent: Pingdom
Allow: /
User-agent: StatusCake
Allow: /
User-agent: Site24x7
Allow: /
User-agent: Freshping
Allow: /
User-agent: HetrixTools
Allow: /

# Translation Services
User-agent: Google Translate
Allow: /
User-agent: bingbot-translate
Allow: /
User-agent: Yandex.Translate
Allow: /

# CDN and Proxy Services
User-agent: Cloudflare
Allow: /
User-agent: AlwaysOnline
Allow: /
User-agent: Wayback Save Page
Allow: /

# Miscellaneous Crawlers
User-agent: Applebot
Allow: /
User-agent: AppleNewsBot
Allow: /
User-agent: Barkrowler
Allow: /
User-agent: BUbiNG
Allow: /
User-agent: Embedly
Allow: /
User-agent: FairShare
Allow: /
User-agent: Grobbot
Allow: /
User-agent: grub-client
Allow: /
User-agent: Holmes
Allow: /
User-agent: HubSpot
Allow: /
User-agent: ICC-Crawler
Allow: /
User-agent: Jooblebot
Allow: /
User-agent: Mail.RU_Bot
Allow: /
User-agent: MauiBot
Allow: /
User-agent: Mediapartners-Google
Allow: /
User-agent: MojeekBot
Allow: /
User-agent: Neevabot
Allow: /
User-agent: PaperLiBot
Allow: /
User-agent: PhantomJS
Allow: /
User-agent: Screaming Frog SEO Spider
Allow: /
User-agent: SeekportBot
Allow: /
User-agent: semanticscholar
Allow: /
User-agent: Siteimprove
Allow: /
User-agent: SMTBot
Allow: /
User-agent: TinEye
Allow: /
User-agent: Veoozbot
Allow: /
User-agent: YaK
Allow: /
User-agent: Zaldamo
Allow: /
User-agent: ZoominfoBot
Allow: /
User-agent: ZumBot
Allow: /

# Sitemap for the lazy scrapers
Sitemap: https://jailbreak.observer/sitemap.xml