Robots.txt file comparison

OldNew
# Google Search Engine Robot # Google Search Engine Robot
# ========================== # ==========================
User-agent: Googlebot User-agent: Googlebot
Allow: /?_escaped_fragment_ Allow: /?_escaped_fragment_
Allow: /*?lang= Allow: /*?lang=
Allow: /hashtag/*?src= Allow: /hashtag/*?src=
Allow: /search?q=%23 Allow: /search?q=%23
Allow: /i/api/ Allow: /i/api/
Disallow: /search/realtime Disallow: /search/realtime
Disallow: /search/users Disallow: /search/users
Disallow: /search/*/grid Disallow: /search/*/grid
Allow: /*?ref_src= Allow: /*?ref_src=
Allow: /*?src= Allow: /*?src=
Disallow: /*? Disallow: /*?
Disallow: /*/followers Disallow: /*/followers
Disallow: /*/following Disallow: /*/following
Disallow: /account/deactivated Disallow: /account/deactivated
Disallow: /settings/deactivated Disallow: /settings/deactivated
# Yahoo! Search Engine Robot
# ==========================
User-Agent: Slurp
Allow: /?_escaped_fragment_
Allow: /*?lang=
Allow: /hashtag/*?src=
Allow: /search?q=%23
Allow: /i/api/
Disallow: /search/realtime
Disallow: /search/users
Disallow: /search/*/grid
Disallow: /*?
Disallow: /*/followers
Disallow: /*/following
Disallow: /account/deactivated
Disallow: /settings/deactivated
# Yandex Search Engine Robot
# ==========================
User-agent: Yandex
Allow: /?_escaped_fragment_
Allow: /*?lang=
Allow: /hashtag/*?src=
Allow: /search?q=%23
Allow: /i/api/
Disallow: /search/realtime
Disallow: /search/users
Disallow: /search/*/grid
Disallow: /*?
Disallow: /*/followers
Disallow: /*/following
Disallow: /account/deactivated
Disallow: /settings/deactivated
# Microsoft Search Engine Robot
# =============================
User-Agent: msnbot
Allow: /?_escaped_fragment_
Allow: /*?lang=
Allow: /hashtag/*?src=
Allow: /search?q=%23
Disallow: /search/realtime
Disallow: /search/users
Disallow: /search/*/grid
Disallow: /*?
Disallow: /*/followers
Disallow: /*/following
Disallow: /account/deactivated
Disallow: /settings/deactivated
# Bing Search Engine Robot
# ========================
User-Agent: bingbot
Allow: /?_escaped_fragment_
Allow: /*?lang=
Allow: /hashtag/*?src=
Allow: /search?q=%23
Disallow: /search/realtime
Disallow: /search/users
Disallow: /search/*/grid
Disallow: /*?
Disallow: /*/followers
Disallow: /*/following
Disallow: /account/deactivated
Disallow: /settings/deactivated
# Every bot that might possibly read and respect this file # Every bot that might possibly read and respect this file
# ======================================================== # ========================================================
User-agent: * User-agent: *
Allow: /*?lang=
Allow: /hashtag/*?src=
Allow: /search?q=%23
Allow: /i/api/
Disallow: /search/realtime
Disallow: /search/users
Disallow: /search/*/grid
Disallow: /*?
Disallow: /*/followers
Disallow: /*/following
Disallow: /account/deactivated
Disallow: /settings/deactivated
Disallow: /oauth
Disallow: /1/oauth
Disallow: /i/streams
Disallow: /i/hello
Disallow: /
# WHAT-4882 - Block indexing of links in notification emails. This applies to all bots. # WHAT-4882 - Block indexing of links in notification emails. This applies to all bots.
# ===================================================================================== # =====================================================================================
Disallow: /i/u Disallow: /i/u
Noindex: /i/u Noindex: /i/u
# Wait 1 second between successive requests. See ONBOARD-2698 for details. # Wait 1 second between successive requests. See ONBOARD-2698 for details.
Crawl-delay: 1 Crawl-delay: 1
# Independent of user agent. Links in the sitemap are full URLs using https:// and need to match # Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
# the protocol of the sitemap. # the protocol of the sitemap.
Sitemap: https://twitter.com/sitemap.xml Sitemap: https://twitter.com/sitemap.xml
Force Refresh Diff