# /robots.txt # this covers most common folder names # adjust as needed per site User-agent: * Disallow: /admin/ Disallow: /common/ Disallow: /css/ Disallow: /download/ Disallow: /images/ Disallow: /includes/ Disallow: /js/ Disallow: /scripts/ Disallow: /templates/ Disallow: *job_search_employees*