# This says to apply these settings to ALL search engine spiders/crawlers
User-agent: *
# Stop spiders from indexing unwanted pages
Disallow: /cgi-bin
Disallow: /cache
Sitemap: http://applerugs.co.uk/sitemapindex.xml