User-agent: ia_archiver Disallow: / # Allow only specific directories User-agent: ScoutJet Disallow: / Allow: /public