# # BSWUSA robots.txt file # # syntax : # User-agent: [spider or bot name] # Disallow: [directory or file name] # Allow: [directory or file name] # Crawl-delay: [delay time in seconds, up to 10][optional, obeyed by some] # Sitemap: http://www.yoursite.com/sitemap.gz [sitemap file location] # # User-agents (*=all robots): # User-agent: * [ALL crawler robots] # User-agent: Slurp [yahoo crawler] # User-agent: Googlebot [google bot crawler] # User-agent: MSNBot [msn bot crawler] # User-agent: Robozilla [DMOZ] # # Disallow: # Disallow: / [disallows root directory, all content] # Disallow: /cgi-bin/ [disallow cgi-bin directory] # Disallow: /filename.asp [disallow specific file by name][wildcards not supported by older bots] # Disallow: /private [disallow ANY path that starts with 'private' at the host root directory (files or subdirs) # # [Other]: # Comments : denoted by hash-mark (#) at beginning of line # Blank Line : indicates a new User-agent section # Capitalization of both commands and bot-names are case-sensitive # Order of execution : robots generally read from top-to-bottom and stop when they reach a section that # applies to them. Thus, any 'User-agent: *' directives should come AFTER specifically named agent-directives. # also, this means that any generally-applied directives must be REPEATED in the '*' directives section # User-agent: * Disallow: /SiteClosed.asp Disallow: /shopcart.asp Disallow: /quote.asp Disallow: /orderverify.asp Disallow: /orderstatus.asp Disallow: /orderprocess.asp Crawl-delay: 10 Sitemap: http://www.bswusa.com/sitemap.xml