#рамблер запретил 23.12.04 16:02:46 на 81.19.66.39 не понимает; 26.12.04 4:37:25 на rambler.ru - StackRambler - нифига не помогает 29.12.04 1:45:18
#Slurp - робот inktomisearch.com Yahoo
#AltaVista: Scooter 
#Infoseek: Infoseek 
#Hotbot: Slurp 
#AOL: Slurp 
#Excite: ArchitextSpider 
#Google: Googlebot 
#Goto: Slurp: 
#Lycos: Lycos 
#MSN: Slurp 
#Netscape: Googlebot 
#NorthernLight: Gulliver 
#WebCrawler: ArchitextSpider 
#Iwon: Slurp 
#Fast: Fast 
#DirectHit: Grabber 
#Yahoo Web Pages: Googlebot <-- Slurp 
#Looksmart Web Pages: Slurp  <-- sv-crawl? sv-crawlfw

#194.165.42.15 - нет Lookup, а жрет трафик на втором мете после Яндекса - троян?
#194.165.42.61
#No host name is associated with this IP address or no reverse lookup is configured.
#Error:Host not found
#194.165.42.61 is from Saudi Arabia(SA) in region Middle East 

#IP address: 94.102.60.154
#  No host name is associated with this IP address or no reverse lookup is configured.  
#  94.102.60.154 is from Netherlands(NL) in region Western Europe 
# - это с форума, могут быть ошибки

#IP address: 78.159.112.189
# No host name is associated with this IP address or no reverse lookup is configured. 
# Error:Timed out 
# 78.159.112.189 is from Germany(DE) in region Western Europe 

#IP address: 92.241.168.207
#No host name is associated with this IP address or no reverse lookup is configured. 
#Error:Host not found 
#92.241.168.207 is from Russian Federation(RU) in region Eastern Europe 


#запрещаем c 15:46 26.11.2008 индексацию под подозрением на трояны (хотя они наверное не смотрят robots.txt) 194.165.42.15 194.165.42.6 78.159.112.189 92.241.168.207


User-agent: 92.241.168.0 - 92.241.168.255   
Disallow: /

User-agent: 78.159.112.0 - 78.159.112.255   
Disallow: /

User-agent: 194.165.42.0 - 194.165.42.255   
Disallow: /

User-agent: 66.90.101.0 - 66.90.101.255  #darhost.com
Disallow: /

############################################

User-agent: sv-crawlfw
Disallow: /

User-agent: sv-crawlfw3
Disallow: /

User-agent: sv-crawl 
Disallow: /

User-agent: Slurp 
Disallow: /xlshop 

User-agent: StackRambler
Disallow: /

User-agent: *
Disallow: /copyhouse/webshop/cgi-bin/index.pl     ### 29.12.04 1:46:02

User-agent: *
Disallow: /webshop.ruweb.net     ### 18.11.05 13:11:48


User-agent: *
Disallow: /webshop.ruweb.net  ###11:28 25.08.2007


User-agent: yandex.ru 
Disallow: /webshop.ruweb.net  ###11:28 25.08.2007
 

#User-agent: META 
#Disallow: /

#User-agent: METASpider 
#Disallow: /

#User-agent: meta-ukraine
#Disallow: /


User-agent: mail.ru         	###2:32 03.07.2010 - не слушает
Disallow: /


User-agent: Mail.Ru         	###18:28 04.07.2010 
Disallow: /


User-agent: aport.ru		###2:32 03.07.2010 - не слушает?
Disallow: /


User-agent: AportWorm		###18:28 04.07.2010
Disallow: /


User-agent: Aport		###18:28 04.07.2010
Disallow: /





#Yahoo:Slurp
User-agent: Slurp
Disallow: /
User-agent: Yahoo
Disallow: /
User-agent: Yahoo Slurp
Disallow: /
User-agent: 72.30.0.0 - 72.30.255.255 
Disallow: /  
User-agent: 74.6.0.0 - 74.6.255.255 
Disallow: /  
#/Yahoo 


User-agent: looksmart
Disallow: /

User-agent: msnbot
Disallow: / 

User-agent: Microsoft
Disallow: /

User-agent: 65.54.188.92
Disallow: /

User-agent: 65.52.0.0 - 65.55.255.255 
Disallow: /

194.165.42.15
User-agent: 65.52.0.0 - 65.55.255.255 
Disallow: /
 
User-agent: *
Disallow: /feedback/
Disallow: /img-index/
Disallow: /ezshop/
Disallow: /banners/
Disallow: /cgi-bin/
#
#для /webshop/ и /arenda/ разрешаем по cgi индексировать демо, кроме admin
Disallow: /webshop/cgi-bin/forum/
Disallow: /webshop/cgi-bin/admin/
Disallow: /webshop/cgi-bin/ch/
Disallow: /webshop/bases/
Disallow: /webshop/img/
Disallow: /webshop/forum_img/
Disallow: /webshop/tmpl/
#
Disallow: /arenda/cgi-bin/
Disallow: /arenda/bases/
Disallow: /arenda/img/
Disallow: /arenda/forum_img/
Disallow: /arenda/tmpl/
#
Disallow: /cosmetics/cgi-bin/
Disallow: /cosmetics/bases/
Disallow: /cosmetics/img/
Disallow: /cosmetics/forum_img/
Disallow: /cosmetics/tmpl/
#
Disallow: /semena/cgi-bin/
Disallow: /semena/bases/
Disallow: /semena/img/
Disallow: /semena/forum_img/
Disallow: /semena/tmpl/ 
#
Disallow: /copyhouse/webshop/cgi-bin/admin/
Disallow: /copyhouse/webshop/cgi-bin/ch/
Disallow: /copyhouse/webshop/bases/
Disallow: /copyhouse/webshop/img/
Disallow: /copyhouse/webshop/forum_img/
Disallow: /copyhouse/webshop/tmpl/
Disallow: /copyhouse/img_index/
Disallow: /copyhouse/webshop/index.html

Disallow: /index-toys.html