# All user-agents are disallowed from crawling standard, 
# non-informational links. These links do not contain secret 
# goodies, but rather they are disallowed to save on bandwidth.

User-agent: Jetbot
Disallow: /

User-agent: TranSGeniKBot
Disallow: /

User-agent: Pompos
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: PhpDig
Disallow: /

User-agent: *
Disallow: /cgi-bin
Disallow: /images