# $Id: robots.txt,v 1.4.4.3 2008/11/04 09:14:25 hass Exp $ # # robots.txt # # This file is to prevent the crawling and indexing of certain parts # of your site by web crawlers and spiders run by sites like Yahoo! # and Google. By telling these "robots" where not to go on your site, # you save bandwidth and server resources. # # This file will be ignored unless it is at the root of your host: # Used: http://example.com/robots.txt # Ignored: http://example.com/site/robots.txt # # For more information about the robots.txt standard, see: # http://www.robotstxt.org/wc/robots.html # # For syntax checking, see: # http://www.sxw.org.uk/computing/robots/check.html User-agent: * # Directories Disallow: /includes/ Disallow: /misc/ Disallow: /modules/ Disallow: /profiles/ Disallow: /images/ Disallow: /scripts/ Disallow: /sites/ Disallow: /themes/ Disallow: /CHANGELOG.txt Disallow: /cron.php Disallow: /INSTALL.mysql.txt Disallow: /INSTALL.pgsql.txt Disallow: /install.php Disallow: /INSTALL.txt Disallow: /LICENSE.txt Disallow: /MAINTAINERS.txt Disallow: /update.php Disallow: /UPGRADE.txt Disallow: /xmlrpc.php Disallow: /admin/ Disallow: /comment/reply/ Disallow: /contact/ Disallow: /logout/ Disallow: /node/add/ Disallow: /search/ Disallow: /user/register/ Disallow: /user/password/ Disallow: /user/login/ Disallow: /?q=admin/ Disallow: /?q=comment/reply/ Disallow: /?q=contact/ Disallow: /?q=logout/ Disallow: /?q=node/add/ Disallow: /?q=search/ Disallow: /?q=user/password/ Disallow: /?q=user/register/ Disallow: /?q=user/login/ Disallow: /*sort= Disallow: *from= Disallow: *sort= Disallow: /*?sort* Disallow: /*&sort* Disallow: /*/track$ Disallow: /*/track?page=* Disallow: /tracker? Disallow: /tracker/ Disallow: /*?quicktabs_* Disallow: /&?quicktabs_* Disallow: /forward?path=* Disallow: /glossary/3#* Disallow: /login?destination= Disallow: /register?destination= Allow: /node/feed Disallow: /taxonomy/ # I want traffic from blogsearch.google. Allow: ////feed # but I don't supply duplicate feeds Disallow: ////0/feed Disallow: ////all/feed Disallow: /taxonomy/term/*/all/feed?page= Disallow: /tagadelic Disallow: /?page Disallow: /author Disallow: /contact Disallow: /logout Disallow: /node/add Disallow: /search/ Disallow: /user/register Disallow: /user/password Disallow: /user/login Disallow: /node$ Disallow: /user$ Disallow: /*/feed$ Disallow: /print/ Disallow: /*/print$ Disallow: /*? User-agent: NimbleCrawler Disallow: / User-agent: Yahoo-MMCrawler Disallow: / User-agent: USyd-NLP-Spider Disallow: / User-agent: MMCrawler Disallow: / User-agent: Shim-Crawler Disallow: / User-agent: MyEngines-Bot Disallow: / User-Agent: KFSW-Bot Disallow: / User-agent: SBIder Disallow: / User-agent: LocalcomBot Disallow: / User-agent: iCCrawler Disallow: / User-agent: KnowItAll Disallow: / User-agent: dcbspider Disallow: / User-agent: Gaisbot Disallow: / User-agent: cfetch Disallow: / User-agent: FindLinks Disallow: / User-agent: BecomeBot Disallow: / User-agent: gonzo1 User-agent: gonzo2 User-agent: gonzo1P User-agent: gonzo2P Disallow: / User-agent: Moni Disallow: / User-agent: Georgios Disallow: / User-agent: CydralSpider Disallow: / User-agent: ObjectsSearch Disallow: / User-agent: HooWWWer Disallow: / User-agent: JemmaTheTourist Disallow: / User-agent: btbot Disallow: / User-agent: Exabot Disallow: / User-agent: Yahoo-MMAudVid Disallow: / User-agent: IRLbot Disallow: / User-agent: BecomeBot Disallow: / User-agent: baiduspider Disallow: / User-agent: Amfibibot Disallow: / User-agent: gridBOT Disallow: / User-agent: sna Disallow: / User-agent: TAMU_CS_IRL_CRAWLER Disallow: / User-agent: NPT Disallow: / User-agent: BruinBot Disallow: / User-agent: ZipppBot Disallow: / User-agent: Jetbot Disallow: / User-agent: Molbsy Disallow: / User-agent: PhpDig Disallow: / User-agent: mirago Disallow: / User-agent: GoForIt.com Disallow: / User-agent: GoForIt Disallow: / User-agent: Download Ninja Disallow: / User-agent: HTTrack Disallow: / User-agent: larbin Disallow: / User-agent: appie Disallow: / User-agent: Yahoo-MMCrawler Disallow: / User-agent: libwww Disallow: / User-agent: SiteSnagger Disallow: / User-agent: WebStripper Disallow: / User-agent: WebCopier Disallow: / User-agent: sohu-search Disallow: / User-agent: heritrix Disallow: / User-agent: WebZIP Disallow: / User-agent: Fetch Disallow: / User-agent: Offline Explorer Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / User-agent: linko Disallow: / User-agent: MSIECrawler Disallow: / User-agent: RPT-HTTPClient Disallow: / User-agent: Dumbot Disallow: / User-agent: Cowbot Disallow: / User-agent: SuperGet Disallow: / #User-agent: ia_archiver #Disallow: / User-agent: psbot Disallow: / User-agent: Szukacz Disallow: / User-agent: antibot Disallow: / User-agent: NaverBot Disallow: / User-agent: Jetbot Disallow: / User-agent: IconSurf Disallow: / User-agent: Speedy Disallow: / User-agent: NPBot Disallow: / User-agent: TutorGig Disallow: / User-agent: Searchspider Disallow: / User-agent: Lachesis Disallow: / User-agent: VSE/1.0 Disallow: / User-agent: QuepasaCreep Disallow: / User-agent: Gigabot Disallow: / User-agent: Haste Disallow: / User-agent: NetResearchServer Disallow: / User-agent: Nutch Disallow: / User-agent: NutchOrg Disallow: / User-agent: grub-client Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 5.0; Windows NT) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows NT) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 95) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 98) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows XP) Disallow: / User-agent: Mozilla/4.0 (compatible; MSIE 4.0; Windows 2000) Disallow: / User-agent: Mozilla/4.0 (Windows XP 5.1) Disallow: / User-agent: Generic Disallow: / User-agent: Steeler Disallow: / User-agent: Ultraseek Disallow: / User-agent: Spinne Disallow: / User-agent: sitecheck.internetseer.com Disallow: / User-agent: :robot Disallow: / User-agent: Spider_ Monkey Disallow: / User-agent: IXE Crawler Disallow: / User-agent: CoolBot Disallow: / User-agent: Zao Disallow: /