Robots.txt file comparison

Previous version copy

  1. # Google Search Engine Robot
  2. # ==========================
  3. User-agent: Googlebot
  4. Allow: /?_escaped_fragment_
  6. Allow: /*?lang=
  7. Allow: /hashtag/*?src=
  8. Allow: /search?q=%23
  9. Allow: /i/api/
  10. Disallow: /search/realtime
  11. Disallow: /search/users
  12. Disallow: /search/*/grid
  14. Allow: /*?ref_src=
  15. Allow: /*?src=
  16. Disallow: /*?
  17. Disallow: /*/followers
  18. Disallow: /*/following
  20. Disallow: /account/deactivated
  21. Disallow: /settings/deactivated
  23. # Yahoo! Search Engine Robot
  24. # ==========================
  25. User-Agent: Slurp
  26. Allow: /?_escaped_fragment_
  28. Allow: /*?lang=
  29. Allow: /hashtag/*?src=
  30. Allow: /search?q=%23
  31. Allow: /i/api/
  32. Disallow: /search/realtime
  33. Disallow: /search/users
  34. Disallow: /search/*/grid
  36. Disallow: /*?
  37. Disallow: /*/followers
  38. Disallow: /*/following
  40. Disallow: /account/deactivated
  41. Disallow: /settings/deactivated
  43. # Yandex Search Engine Robot
  44. # ==========================
  45. User-agent: Yandex
  46. Allow: /?_escaped_fragment_
  48. Allow: /*?lang=
  49. Allow: /hashtag/*?src=
  50. Allow: /search?q=%23
  51. Allow: /i/api/
  52. Disallow: /search/realtime
  53. Disallow: /search/users
  54. Disallow: /search/*/grid
  56. Disallow: /*?
  57. Disallow: /*/followers
  58. Disallow: /*/following
  60. Disallow: /account/deactivated
  61. Disallow: /settings/deactivated
  63. # Microsoft Search Engine Robot
  64. # =============================
  65. User-Agent: msnbot
  66. Allow: /?_escaped_fragment_
  68. Allow: /*?lang=
  69. Allow: /hashtag/*?src=
  70. Allow: /search?q=%23
  71. Disallow: /search/realtime
  72. Disallow: /search/users
  73. Disallow: /search/*/grid
  75. Disallow: /*?
  76. Disallow: /*/followers
  77. Disallow: /*/following
  79. Disallow: /account/deactivated
  80. Disallow: /settings/deactivated
  82. # Bing Search Engine Robot
  83. # ========================
  84. User-Agent: bingbot
  85. Allow: /?_escaped_fragment_
  87. Allow: /*?lang=
  88. Allow: /hashtag/*?src=
  89. Allow: /search?q=%23
  90. Disallow: /search/realtime
  91. Disallow: /search/users
  92. Disallow: /search/*/grid
  94. Disallow: /*?
  95. Disallow: /*/followers
  96. Disallow: /*/following
  98. Disallow: /account/deactivated
  99. Disallow: /settings/deactivated
  101. # Every bot that might possibly read and respect this file
  102. # ========================================================
  103. User-agent: *
  104. Allow: /*?lang=
  105. Allow: /hashtag/*?src=
  106. Allow: /search?q=%23
  107. Allow: /i/api/
  108. Disallow: /search/realtime
  109. Disallow: /search/users
  110. Disallow: /search/*/grid
  112. Disallow: /*?
  113. Disallow: /*/followers
  114. Disallow: /*/following
  116. Disallow: /account/deactivated
  117. Disallow: /settings/deactivated
  119. Disallow: /oauth
  120. Disallow: /1/oauth
  122. Disallow: /i/streams
  123. Disallow: /i/hello
  125. # WHAT-4882 - Block indexing of links in notification emails. This applies to all bots.
  126. # =====================================================================================
  127. Disallow: /i/u
  128. Noindex: /i/u
  130. # Wait 1 second between successive requests. See ONBOARD-2698 for details.
  131. Crawl-delay: 1
  133. # Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
  134. # the protocol of the sitemap.
  135. Sitemap: https://twitter.com/sitemap.xml

New version copy

  1. # Google Search Engine Robot
  2. # ==========================
  3. User-agent: Googlebot
  4. Allow: /?_escaped_fragment_
  6. Allow: /*?lang=
  7. Allow: /hashtag/*?src=
  8. Allow: /search?q=%23
  9. Allow: /i/api/
  10. Disallow: /search/realtime
  11. Disallow: /search/users
  12. Disallow: /search/*/grid
  14. Allow: /*?ref_src=
  15. Allow: /*?src=
  16. Disallow: /*?
  17. Disallow: /*/followers
  18. Disallow: /*/following
  20. Disallow: /account/deactivated
  21. Disallow: /settings/deactivated
  24. # Every bot that might possibly read and respect this file
  25. # ========================================================
  26. User-agent: *
  27. Disallow: /
  29. # WHAT-4882 - Block indexing of links in notification emails. This applies to all bots.
  30. # =====================================================================================
  31. Disallow: /i/u
  32. Noindex: /i/u
  34. # Wait 1 second between successive requests. See ONBOARD-2698 for details.
  35. Crawl-delay: 1
  37. # Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
  38. # the protocol of the sitemap.
  39. Sitemap: https://twitter.com/sitemap.xml