#All robots will spider the domain User-agent: * Disallow: /template.html Disallow: /indextest.html Disallow: /index-test.html Disallow: /indexaaa.html