# this file tells robots which directories/files to ignore. User-agent: * Disallow: /unique/ User-agent: GoogleBot Disallow: /unique/