User-agent: *
Sitemap: https://habr.com/sitemap/index.xml

User-agent: Yandex
Crawl-delay: 2
Disallow: /search/
Disallow: /js/
Disallow: /css/
Disallow: /ajax/
Disallow: /login/
Disallow: /register/
Disallow: /ru/viewcount/
Disallow: /en/viewcount/
Clean-param: utm_source&utm_medium&utm_term&utm_campaign /

User-agent: Googlebot
Disallow: /search/
Disallow: /js/
Disallow: /css/
Disallow: /ajax/
Disallow: /login/
Disallow: /register/
Disallow: /ru/viewcount/
Disallow: /en/viewcount/

User-agent: Slurp
Crawl-delay: 8
Disallow: /search/
Disallow: /js/
Disallow: /css/
Disallow: /ajax/
Disallow: /login/
Disallow: /register/
Disallow: /ru/viewcount/
Disallow: /en/viewcount/

User-agent: *
Crawl-delay: 10
Disallow: /search/
Disallow: /js/
Disallow: /css/
Disallow: /ajax/
Disallow: /login/
Disallow: /register/
Disallow: /ru/viewcount/
Disallow: /en/viewcount/