User-agent: *  # applies to all robot agents
Disallow:	# allows everything to be indexed

Crawl-delay: 3 # Number of seconds to wait between successive requests

Request-rate: 1/4 # maximum rate is one page every 4 seconds