User-agent: *

Disallow: /test
Disallow: /cgi-bin
Disallow: *.txt
Disallow: *.zip
Disallow: *.pdf