#All robots can spider the domain User-agent: * Disallow: #Disallow images and frontpage extensions User-agent: * Disallow: /images Disallow: /_private Disallow: /_vti_cnf Disallow: /_vti_bin Disallow: /_vti_log Disallow: /_vti_pvt Disallow: /_vti_txt