User-agent: *

###cms
Disallow: /admin/ 
Disallow: /contrib/ 
Disallow: /doc/ 
Disallow: /lib/ 
Disallow: /modules/ 
Disallow: /plugins/ 
Disallow: /scripts/ 
Disallow: /tmp/
###end cms
Disallow: /favicon.ico
Disallow: /incl/
Disallow: /images/
Disallow: /cgi-bin/
Disallow: /gallery/images/
Disallow: /*.gif$
Disallow: /*.jpg$
Disallow: /*.jpeg$
Disallow: /*.png$
Disallow: /*?

Crawl-delay: 20





####selectively not ok bot#######

#User-agent: Alexibot #ia_archiver (www.alexa.com)

User-agent: Aqua_Products

User-agent: Atomz

User-agent: aipbot

User-agent: Arikus_Spider

User-agent: asterias

User-agent: AVSearch

User-agent: BackStreet Browser

User-agent: BackDoorBot/1.0

User-agent: Baiduspider

User-agent: BDFetch

User-agent: BecomeBot

User-agent: Black Hole

User-agent: Bookmark search tool

User-agent: BlowFish

User-agent: BlowFish/1.0

User-agent: BotALot

User-agent: BuiltBotTough

User-agent: Bullseye

User-agent: BunnySlippers

User-agent: b2w/0.1

User-agent: Cegbfeieh

User-agent: Charlotte

User-agent: CheeseBot

User-agent: CherryPicker

User-agent: CherryPickerSE/1.0

User-agent: CherryPickerElite/1.0

User-agent: ConveraCrawler

User-agent: ConveraCrawler/0.4

User-agent: ConveraCrawler/0.9d

User-agent: Copernic

User-agent: CopyRightCheck 

User-agent: Crescent

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0

User-agent: cosmos

User-agent: CosmixCrawler

User-agent: CosmixCrawler/0.1

User-agent: CrawLynx

User-agent: Curl

User-agent: DA 5.3

User-agent: DittoSpyder

User-agent: dloader(NaverRobot)/1.0

User-agent: dumbot

User-agent: EmailCollector

User-agent: EmailSiphon

User-agent: EmailWolf

User-agent: Enterprise_Search/1.0

User-agent: Enterprise_Search

User-agent: EroCrawler

User-agent: es

User-agent: ExtractorPro

User-agent: FairAd Client

User-agent: FAST MetaWeb Crawler

User-agent: FAST Enterprise Crawler

User-agent: FAST-WebCrawler

User-agent: FAST-WebCrawler/3.8 (crawler at trd dot overture dot com; http://www.alltheweb.com/help/webmaster/crawler)

User-agent: Flaming AttackBot

User-agent: flashget

User-agent: Firefly

User-agent: Franklin Locator 1.8

User-agent: Foobot

User-agent: Gaisbot

User-agent: Gigabot

User-agent: GOFORITBOT

User-agent: grub-client

User-agent: grub

User-agent: g2Crawler

User-agent: Hatena Antenna

User-agent: Harvest

User-agent: heritrix

User-agent: hloader€

User-agent: hl_ftien_spider

User-agent: hloader

User-agent: htdig

User-agent: humanlinks

User-agent: HTTrack

User-agent: humanlinks

User-agent: httplib

User-agent: InfoNaviRobot

User-agent: InfoSeek Sidewinder/0.9

User-agent: Indy Library

User-agent: Iron33/1.0.2

User-agent: IRLbot/1.0

User-agent: lwp-trivial

User-agent: JennyBot

User-agent: Jetbot

User-agent: Kenjin Spider

User-agent: Keyword Density

User-agent: larbin

User-agent: LexiBot

User-agent: libWeb/clsHTTP

User-agent: libwww-perl

User-agent: LinkextractorPro

User-agent: LinkScan

User-agent: LinkScan/8.1a Unix

User-agent: LinkWalker

User-agent: LNSpiderguy

User-agent: looksmart

User-agent: Lycos_Spider

User-agent: Lynx

User-agent: Mata Hari

User-agent: Microsoft URL Control

User-agent: Microsoft URL Control - 6.00.8169

User-agent: Microsoft URL Control - 5.01.4511

User-agent: Mercator

User-agent: MIIxpc

User-agent: MIIxpc/4.2

User-agent: Mister PiX

User-agent: moget

User-agent: MSIECrawler

User-agent: MultiText

User-agent: naver

User-agent: NetAnts

User-agent: NetMechanic

User-agent: NICErsPRO

User-agent: Nimblecrawler

User-agent: NCSU CRAWLER

User-agent: nicebot

User-agent: noxtrumbot

User-agent: NG

User-agent: NutchCVS

User-agent: Offline Explorer

User-agent: Openbot/3.0

User-agent: Openfind

User-agent: Openfind data gatherer

User-agent: Oracle Ultra Search

User-agent: Pita

User-agent: PerMan

User-agent: POE-Component-Client

User-agent: Pompos

User-agent: ProPowerBot

User-agent: ProWebWalker

User-agent: psbot

User-agent: psbot/0.1 (+http://www.picsearch.com/bot.html)

User-agent: Python-urllib

User-agent: QuepasaCreep

User-agent: QueryN Metasearch

User-agent: Radiation Retriever 1.1

User-agent: RepoMonkey

User-agent: RepoMonkey Bait & Tackle/v1.01

User-agent: RMA

User-agent: SBider

User-agent: Scooter

User-agent: Scooter-ARS-1.1

User-agent: Scooter/3.2

User-agent: Scooter/3.3

User-agent: Scooter/3.3_SF

User-agent: SearchengineLicenceSheep

User-agent: searchpreview

User-agent: SearchMechanics

User-agent: shadow

User-agent: SiteSnagger

User-agent: SiteSnagger

User-agent: Skepee

User-agent: Slurp

User-agent: SpankBot

User-agent: spanner

User-agent: sproose

User-agent: SurveyBot

User-agent: suzuran

User-agent: Synoobot

User-agent: Szukacz

User-agent: teleport

User-agent: TeleportPro

User-agent: Telesoft

User-agent: The Intraformant

User-agent: TheNomad

User-agent: TightTwatBot

User-agent: Titan

User-agent: The Intraformant

User-agent: toCrawl/UrlDispatcher

User-agent: True_Robot

User-agent: turingos

User-agent: TV33_Mercator

User-agent: Ultraseek

User-agent: URL Control

User-agent: URL_Spider_Pro

User-agent: URLy Warning

User-agent: VCI

User-agent: WebCapture

User-agent: webdup

User-agent: WebAuto

User-agent: WebBandit

User-agent: WebCopier

User-agent: WebEnhancer

User-agent: WebGather 3.0

User-agent: Web Image Collector

User-agent: WebmasterWorld Extractor

User-agent: WebmasterWorldForumBot

User-agent: WebSauger

User-agent: Website Quester

User-agent: Webster Pro

User-agent: WebStripper

User-agent: WebZip

User-agent: Wget

User-agent: Wget/1.6

User-agent: Wget/1.5.3

User-agent: wire

User-agent: WWW-Collector-E

User-agent: www.webwombat.com.au

User-agent: VCI

User-agent: VCI WebViewer VCI WebViewer Win32

User-agent: Xenu

User-agent: Xenu's Link Sleuth 1.1c

User-agent: Xenu's

User-agent:  Yahoo-NewsCrawler*

User-agent: Yahoo-MMCrawler/3.x	

User-agent: zspider

User-agent: Zeus

User-agent: Zeus Link Scout

User-agent: Zeus 32297 Webster Pro V2.9 Win32

User-agent: ZyBorg

User-agent: ZyBorg/1.0



Disallow: /



####selectively ok bot#######
User-Agent: Googlebot
User-agent: Googlebot-Image  #google's image bot 
User-agent: Mediapartners-Google*  ###Google Adsense
User-agent: ia_archiver
User-agent: IconSurf
User-agent: internetseer.com
User-agent: msnbot
User-agent: mozDexOrg  ###from the dmoz.org

Disallow: /admin/ 
Disallow: /contrib/ 
Disallow: /doc/ 
Disallow: /lib/ 
Disallow: /modules/ 
Disallow: /plugins/ 
Disallow: /scripts/ 
Disallow: /tmp/

Disallow: /favicon.ico
Disallow: /cgi-bin/
Disallow: /incl/
Disallow: /images/
Disallow: /gallery/images/
Disallow: /*.gif$
Disallow: /*.jpeg$
Disallow: /*.jpg$
Disallow: /*.png$
Disallow: /*?