#Crawl-delay directive
User-agent: *
Crawl-delay: 1000


User-agent: * 
Disallow: /catalog/wosadmin/
Disallow: /catalog/cache/
Disallow: /catalog/components/
Disallow: /catalog/editor/
Disallow: /catalog/help/
Disallow: /catalog/images/
Disallow: /catalog/includes/
Disallow: /catalog/language/
Disallow: /catalog/mambots/
Disallow: /catalog/media/
Disallow: /catalog/modules/
Disallow: /catalog/templates/
Disallow: /installation/
Disallow: /ypslideoutmenus/
Disallow: /catalog/includes/
Disallow: /cgi-bin/
Disallow: /errors/
Disallow: /images/
Disallow: /library/
Disallow: /manager/
Disallow: /plugins/
Disallow: /users/
Disallow: /pma/
Disallow: /transfer/
Disallow: /wpm/
Disallow: /index.html
Disallow: /manager.html
Disallow: /catalog/account.php
Disallow: /catalog/account_edit.php
Disallow: /catalog/account_history.php
Disallow: /catalog/account_history_info.php
Disallow: /catalog/account_password.php
Disallow: /catalog/add_checkout_success.php
Disallow: /catalog/address_book.php
Disallow: /catalog/address_book_process.php
Disallow: /catalog/advanced_search.php
Disallow: /catalog/advanced_search_result.php
Disallow: /catalog/checkout_confirmation.php
Disallow: /catalog/checkout_payment.php
Disallow: /catalog/checkout_payment_address.php
Disallow: /catalog/checkout_process.php
Disallow: /catalog/checkout_shipping.php
Disallow: /catalog/checkout_shipping_address.php
Disallow: /catalog/checkout_success.php
Disallow: /catalog/contact_dc.php
Disallow: /catalog/contact_us.php
Disallow: /catalog/cookie_usage.php
Disallow: /catalog/create_account.php
Disallow: /catalog/create_account_success.php
Disallow: /catalog/login.php
Disallow: /catalog/password_forgotten.php
Disallow: /catalog/popup_image.php
Disallow: /catalog/shopping_cart.php
Disallow: /catalog/product_reviews_write.php
Disallow: /catalog/submit_testimonial.php
Disallow: /catalog/testimonial_contact_us.php
Disallow: /catalog/tell_a_friend.php
Disallow: /catalog/stylesheet.css
Disallow: /catalog/stylesheet_personal_shopper.css
Disallow: /catalog/sts_stylesheet.css
Disallow: /catalog/sts_styles.css
Disallow: /catalog/personal_shopper_form_thankyou.php
Disallow: /catalog/links_submit.php
Disallow: /catalog/links_submit_success.php 
Disallow: /stylesheet.css
Disallow: /catalog/catalog/images/sky_2.jpg 
Disallow: /favicon.ico



#Items AVM found in stats files

#Picsearch indexes images from the web. To do this we use a web-crawler which identifies itself as 'Psbot'.
User-agent: psbot
Disallow: /

#The boitho.com-robot is a distributed crawler that downloads web pages to build the database used by boitho.com to search in. 
User-agent: boitho.com-dc
Disallow: /


User-agent: BaiDuSpider
Disallow: /

User-agent: Nutch
Disallow: /