robotsTxtGenerator.name

robotsTxtGenerator.description

robotsTxtGenerator.userAgent

robotsTxtGenerator.directives

robotsTxtGenerator.sitemapUrl

robotsTxtGenerator.crawlDelay

robotsTxtGenerator.hostUrl

robotsTxtGenerator.charactersCount

robotsTxtGenerator.uploadStep

robotsTxtGenerator.accessStep

robotsTxtGenerator.testStep

robotsTxtGenerator.monitorStep

Disallow: /admin/ Disallow: /private/

Disallow: /*.pdf$ Disallow: /*.doc$

User-agent: Googlebot Allow: /

User-agent: * Crawl-delay: 10