# Policies for all agents
User-agent: *
# keep robots out of the cgi & archive areas
Disallow: /comics/
Disallow: /cgi-bin/
Disallow: /mailman/
Disallow: /horde/
Disallow: /kplaylist/
Disallow: /musik/
Disallow: /rog/
Disallow: /test/
Disallow: /ups/
Disallow: /data/
Disallow: /archives/
Disallow: /Templates/
Disallow: /beos/data/
Disallow: /rig/distrib/
# these generate too much traffic and are useless to index
Disallow: /photos
Disallow: /ballade
# keep robots out of zip files and dynamic stuff
Disallow: /*.tgz
Disallow: /*.gz
Disallow: /*.zip
Disallow: /*.cgi
Disallow: /*.php
# and rss
Disallow: =rss

# keep some rude engines out of the way
User-agent: grub-client
Disallow: /
User-agent: OmniExplorer_Bot
Disallow: /
User-agent: BecomeBot
Disallow: /
User-agent: psbot
Disallow: /

# RM 20060505 reduce activity
User-agent: Slurp
Crawl-delay: 120
User-agent: msnbot
Crawl-delay: 3600