User-agent: * Disallow: /cgi-bin/ # exclude robots from specified tree Disallow: /backup/ Disallow: /ftp/ Disallow: /data/ Disallow: /Library/ Disallow: /bilder/ Disallow: /logs/ User-agent: ia_archiver Disallow: /