#
# robots.txt 
# Created: Mon, 21 Jun 2010 15:16:37 GMT
#
# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there. If you're
# irresponsible, your access to the site may be blocked.
#

User-agent: psbot
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: ScoutJet
Disallow: /

User-agent: wget
Disallow: /

User-agent: Eurobot
Disallow: /

# http://www.cuil.com/info/webmaster_info/
User-agent: twiceler
Disallow: /
User-agent: twiceler.
Disallow: /
User-agent: "twiceler."
Disallow: /

User-agent: Gaisbot
Disallow: /

# http://www.dotnetdotcom.org/
User-agent: dotbot 
Disallow: / 

User-agent: WWW-Mechanize
Disallow: /

# http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
User-agent: MJ12bot
Disallow: /

# http://www.suchen.de/popups/faq.jsp
User-agent: gonzo*
Disallow: /
User-agent: gonzo
Disallow: /

User-agent: SapphireWebCrawler
Disallow: /

# http://www.amfibi.com/cabot/
User-agent: Cabot
Disallow: / 

# http://spider.acont.de/
User-agent: ACONTBOT
Disallow: /

# http://turnitin.com/robot/crawlerinfo.html
User-agent: TurnitinBot
Disallow: /

User-agent: CatchBot
Disallow: /

User-agent: WebRankSpider 
Disallow: /

User-agent: yacy
Disallow: /

User-agent: yacybot
Disallow: /

User-agent: Mail.Ru
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: SurveyBot_IgnoreIP 
Disallow: / 

User-agent: Yanga WorldSearch Bot
Disallow: /

#http://www.setooz.com/oozbot.html 
User-agent: OOZBOT
Disallow: /

#http://www.botje.com/plukkie.htm
User-agent: plukkie
Disallow: / 

User-agent: http://www.uni-koblenz.de/~flocke/robot-info.txt
Disallow: / 

User-agent: Naver
Disallow: / 
User-agent: NaverBot
Disallow: / 
User-agent: Yeti
Disallow: / 

User-Agent: iisbot
Disallow: /

User-agent: Gigabot
Disallow: /

# http://www.mojeek.com/bot.html
User-agent: MojeekBot
Disallow: /

User-agent: citenikbot
Disallow: /

User-Agent: Charlotte
Disallow: /

# http://www.exabot.com/go/robot
User-agent: Exabot
Disallow: /

# 403	http://robot.vedens.de
User-agent: VEDENSBOT
Disallow: /

User-agent: Lexxebot
Disallow: /

User-agent: VoilaBot
Disallow: /

#  http://help.soso.com/webspider.htm
User-agent: Sosospider
Disallow: /
User-agent: "Sosospider"
Disallow: /


User-agent: Tagoobot
Disallow: /

#  http://www.cityreview.org/crawler
User-agent: cityreview
Disallow: /

User-agent: Euripbot
Disallow: /

User-Agent: Butterfly
Disallow: /

User-agent: isara-search
Disallow: /

User-agent: Jyxobot
Disallow: /

# http://www.80legs.com/spider.html
User-agent: 008
Disallow: /

# 403 specialists -------------------
User-agent: MLBot
Disallow: / 

User-agent: libwww-perl
Disallow: /

User-agent: Nutch
Disallow: /
User-agent: nutch-agent
Disallow: /

User-agent: panscient.com
Disallow: /

User-agent: BotOnParade
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: jobs.de-Robot
Disallow: /

User-agent: Clewwa-Bot
Disallow: /


#http://www.search17.com/bot.php
User-agent: search17
Disallow: /

User-agent: spbot
Disallow: /

# 403 http://spinn3r.com/robot
User-agent: Spinn3r
Disallow: /

# http://www.entireweb.com/about/search_tech/speedy_spider/		Entireweb Robot
User-agent: Speedy
Disallow: /


User-agent: CatchBot
Disallow: /

User-agent: ccbot
Disallow: /

# http://www.search17.com/bot.php
User-agent: search17
Disallow: /

# http://www.envolk.com/envolkspiderinfo.html
User-agent: envolk
Disallow: /

# http://www.wise-guys.nl/webcrawler.php?item=crawlers
User-agent: Vagabondo
Disallow: /
User-agent: Bilbo
Disallow: /

# http://www.tineye.com/faq
User-agent: TinEye
Disallow: /
User-agent: "TinEye"
Disallow: /


User-Agent: gsa-crawler
Disallow: /

User-agent: bixolabs
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: *
Disallow: /netz/
Disallow: *NOINDEX*
Disallow: /controls/
Disallow: /App_Themes/
Disallow: /WebResource.axd
Disallow: /login.aspx