# The Coup Calgary - Robots.txt # https://www.thecoup.ca # Last Updated: January 2025 # =========================================== # Default rules for all crawlers # =========================================== User-agent: * Allow: / # Disallow utility and backend directories Disallow: /config/ Disallow: /search/ Disallow: /account/ Disallow: /api/ Disallow: /static/ # Disallow query parameters that create duplicate content Disallow: /*?*author=* Disallow: /*?*tag=* Disallow: /*?*month=* Disallow: /*?*view=* Disallow: /*?*format=* # Disallow form pages (not for indexing) Disallow: /reservation.html Disallow: /enquiry.html # Disallow legal pages from general crawling Disallow: /privacy-policy.html Disallow: /terms-of-use.html # Disallow component files (not meant to be accessed directly) Disallow: /components/ # =========================================== # Search Engine Crawlers - Block llms.txt from indexing # =========================================== User-agent: Googlebot Disallow: /llms.txt Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: Googlebot-Image Allow: /assets/ User-agent: Bingbot Disallow: /llms.txt Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: Slurp Disallow: /llms.txt Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: DuckDuckBot Disallow: /llms.txt Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: Yandex Disallow: /llms.txt Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ # =========================================== # AI Crawlers - Allow access to llms.txt # =========================================== User-agent: GPTBot Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: ChatGPT-User Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: anthropic-ai Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: Claude-Web Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: CCBot Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: Google-Extended Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: cohere-ai Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: PerplexityBot Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: Amazonbot Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ User-agent: FacebookBot Allow: /llms.txt Allow: / Disallow: /reservation.html Disallow: /enquiry.html Disallow: /privacy-policy.html Disallow: /terms-of-use.html Disallow: /components/ # =========================================== # Google Ads Bots # =========================================== User-agent: AdsBot-Google User-agent: AdsBot-Google-Mobile User-agent: AdsBot-Google-Mobile-Apps Allow: / # =========================================== # Crawl delay for aggressive bots # =========================================== User-agent: Baiduspider Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 5 User-agent: AhrefsBot Crawl-delay: 5 # =========================================== # Sitemap Location # =========================================== Sitemap: https://www.thecoup.ca/sitemap.xml # =========================================== # Notes # =========================================== # - llms.txt is allowed for AI crawlers but blocked from search engine indexing # - reservation.html and enquiry.html are form pages not meant for indexing # - Legal pages (privacy-policy.html, terms-of-use.html) are excluded from crawling # - Component files are internal templates, not standalone pages