# Disallow directories User-agent: * Disallow: /producers/ Disallow: /cgi-bin/ Disallow: /test/ Disallow: /link1.html Disallow: /link2.html Disallow: /link3.html # Disallow known harmful web bots User-agent: WebCopier v4.3 Disallow: / # Googlebot User-agent: Googlebot Disallow: /producers/ Disallow: /cgi-bin/ Disallow: /test/ Disallow: /private/ Disallow: /test/ Disallow: /link1.html Disallow: /link2.html Disallow: /link3.html