# Make changes for all web spiders
User-agent: htdig
Disallow:

User-agent: *
Disallow: / 

Crawl-delay: 120