# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
Host: rubyclub.com.ua
Sitemap: http://rubyclub.com.ua/sitemap.xml
#User-agent:  Yahoo! Slurp    //For yahoo bot
#Disallow: /

#User-agent: msnbot  // For MSN bot
#Disallow: /

#User-agent: ArchitextSpider  //For Exite bot
#Disallow: /

#User-agent: FAST //For Alltheweb bot
#Disallow: /

#User-agent: Scooter  //For Altavista Bot
#Disallow: /

#User-agent: ia_archiver //For Alexa Bot
#Disallow: /
# allow adsense bot on entire site
#User-agent: Mediapartners-Google*
#Disallow: /

User-agent:   *
Disallow: /messages/tag_by_id
Disallow: /users/
Disallow: /gem/update_table/
Disallow: /doc/active_scaffold/
Disallow: /doc/forum-api/


User-agent: Slurp
Crawl-delay: 3.0

User-agent: MSNBot
Crawl-delay: 0.5