Allow: /

# Disallow specific bots from indexing or crawling the site at all
# Most recent additions first.

User-agent: GigaBot
Disallow: /

User-agent: Voyager
Disallow: /

User-agent: BaiDuSpider
Disallow: /

User-agent: BackRub
Disallow: /

User-agent: Grub.org
Disallow: /

User-agent: BotRightHere
Disallow: /

User-agent: larbin
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Walhello appie
Disallow: /

User-agent: Python-urllib
Disallow: /


User-agent: CherryPicker
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: yandex bot
Disallow: /

#Specific file disallow

User-Agent: *

Disallow: /language/en-us/
Disallow: /*?st=
Disallow: */st/1/id/+.aspx*
Disallow: */Network-Status-List.aspx?service*
Disallow: */NetworkStatusList.aspx?service*
Disallow: */NetworkStatusFaultList.aspx?Id=*
Disallow: */Network-Status-Fault-List.aspx?Id=*
Disallow: */Secure/Download.aspx?file=*
Disallow: /Forgotten-Password.aspx
Disallow: /Secure.aspx
Disallow: /Secure/Download.aspx
Disallow: */statusservice.aspx?*
Disallow: */ctl/*
Disallow: /Portals/0/Video/
Disallow: /Default.aspx?TabId=242
Disallow: /tabid/242/Default.aspx
Disallow: /LinkClick.aspx
# END OF FILE