User-agent: * Disallow: /confused/ # User-agent: * # Disallow: /checker/clickthrough.cgi # http://www.picsearch.com/ User-agent: psbot Disallow: / # http://www.cuill.com/twiceler/robot.html User-agent: twiceler Disallow: / # yetibot@naver.com User-agent: Yeti Disallow: / # http://www.netseer.com/crawler.html User-agent: Teemer Disallow: / # dunno who, someone in .cz User-agent: Jyxobot Disallow: / # http://www.cazoodle.com User-agent: CazoodleBot Disallow: / # http://www.majestic12.co.uk/projects/dsearch/mj12bot.php User-agent: MJ12bot Disallow: / # http://www.baidu.com/search/spider.htm User-agent: Baiduspider Disallow: / # http://www.192.com/support/web-crawler.cfm User-agent: 192.comAgent Disallow: / # http://discoveryengine.com/discobot.html User-agent: discobot Disallow: / # http://reap.cs.cmu.edu/REAP-crawler/ User-agent: REAP-crawler Disallow: / User-agent: Nutch Disallow: / User-agent: Stevan/Nutch Disallow: / # http://www.kalooga.com User-agent: Kalooga Disallow: / # http://www.seekport.co.uk/help/webmaster_robot.html User-agent: seekbot Disallow: / # we give Oddle a feed, no need for them to scrape us too... User-agent: oodlebot Disallow: / # www.scoutjet.com User-agent: ScoutJet Disallow: / # www.webalta.net User-agent: WebAlta Disallow: / # adshadow.de User-agent: AdShadow Disallow: / # Internet Research Institute http://iri-uk.com User-agent: 3GSE Disallow: / # http://gais.cs.ccu.edu.tw/robot.php User-agent: Gaisbot Disallow: / # DotBot http://www.dotnetdotcom.org User-agent: DotBot Disallow: / # http://www.80legs.com/spider.html User-agent: 008 Disallow: / # http://www.80legs.com/spider.html User-agent: 80legs Disallow: / # http://www.commoncrawl.org/faq.htm User-agent: CCBot Disallow: / # http://www.puritysearch.net/ User-agent: Purebot Disallow: / # http://www.botje.com/plukkie.htm User-agent: plukkie Disallow: / # http://www.search17.com/bot.php User-agent: search17 Disallow: / # http://yacy.net/bot.html User-agent: yacybot Disallow: /