# robots.txt for osCommerce sites - IGP08

# Feed sitemap to crawlers that understand it
Sitemap: http://www.style-exchange.co.uk/sitemap.php

# Don't allow Google Images in at all
#User-agent: Googlebot-Image
#Disallow: /

# Apply the below to ALL search engine spiders/crawlers, these 
# pages have no benefit to/from search engine results.
User-agent: *
Disallow: /admin
Disallow: /account.php
Disallow: /account_edit.php
Disallow: /account_history.php
Disallow: /account_history_info.php
Disallow: /account_newsletters.php
Disallow: /account_notifications.php
Disallow: /account_password.php
Disallow: /add_checkout_success.php
Disallow: /address_book.php
Disallow: /address_book_process.php
Disallow: /advanced_search.php
Disallow: /advanced_search_result.php
Disallow: /checkout_confirmation.php      
Disallow: /checkout_payment.php
Disallow: /checkout_payment_address.php
Disallow: /checkout_process.php
Disallow: /checkout_shipping.php
Disallow: /checkout_shipping_address.php
Disallow: /checkout_success.php
Disallow: /create_account.php
Disallow: /create_account_success.php
Disallow: /download.php
Disallow: /email_a_friend.php
Disallow: /gv_redeem.php
Disallow: /gv_send.php
Disallow: /gv_faq.php
Disallow: /info_shopping_cart.php
Disallow: /login.php
Disallow: /logoff.php
Disallow: /password_forgotten.php
Disallow: /popup_image.php
Disallow: /popup_search_help.php
Disallow: /print_order.php
Disallow: /print_order_2.php
Disallow: /product_reviews_write.php
Disallow: /redirect.php
Disallow: /shipping.php
Disallow: /shopping_cart.php
Disallow: /ssl_check.php
Disallow: /wpcallback.php

# Make bots that respect it limit hits to 1 per 10sec
User-agent: *
Crawl-delay: 10

# Block advertising-related bots:
User-agent: Mediapartners-Google*
Disallow: /

# Block web site downloaders

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: wget
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: k2spider
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /

# Don't allow the wayback-machine to index pages
User-agent: ia_archiver
Disallow: /