# Identify sitemap index file, this has google sitemap and google news sitemap 
Sitemap: http://www.dma.org.uk/sitemap_index.xml

# Disallow Google ad bots from the entire site 
User-agent: Mediapartners-Google 
Disallow: / 

# Disallow Google Adsense from the entire site 
User-agent: Google AdSense 
Disallow: / 

# Disallow specific bots from indexing or crawling the site at all 
User-agent: Gigabot 
Disallow: / 

User-agent: Voyager 
Disallow: / 

User-agent: BaiDuSpider 
Disallow: / 

User-agent: BackRub 
Disallow: / 

User-agent: Grub.org 
Disallow: / 

User-agent: botRightHere 
Disallow: / 

User-agent: larbin 
Disallow: / 

User-agent: psbot 
Disallow: / 

User-agent: Walhello appie 
Disallow: / 

User-agent: Python-urllib 
Disallow: / 

User-agent: Googlebot-Image 
Disallow: / 

User-agent: CherryPicker 
Disallow: / 

User-agent: EmailCollector 
Disallow: / 

User-agent: WebBandit 
Disallow: / 

User-agent: EmailWolf 
Disallow: / 

User-agent: CopyRightCheck 
Disallow: / 

User-agent: Crescent 
Disallow: / 

User-agent: yandex bot 
Disallow: / 

User-agent: YandexBot 
Disallow: / 

User-agent: ScoutJet 
Disallow: / 

# Disallow all from these directories and all files within them 
User-agent: * 
Disallow: /_Admin 
Disallow: /_private 
Disallow: /_shared 
Disallow: /compliance 
Disallow: /styles 
Disallow: /privacy 
Disallow: /login 
Disallow: /dropped 
Disallow: /App-Code