# allow valid crawlers
User-agent: Adsbot  # Google Ads
User-agent: Googlebot  # Google search
User-agent: bingbot  # Microsoft bing
User-agent: Twitterbot  # Twitterbot
User-agent: Facebot  # Facebook
User-agent: Applebot  # Apple
Disallow:
Disallow: /users/get_ajax_forgot
Disallow: */trade_offers/ajax_send_mail_to_unclear_user/
Disallow: */products/get_favorited_users_popup/
Disallow: */products/get_premium_item_popup/
Disallow: /category/clothing?filtred=1&category_id=5&parent_category_id=1&browse_category=1&shoe_category_id=2&bag_category_id=3
Disallow: /critical_errors/error500
Disallow: /error404
# a valid crawler does not need the following files
Disallow: /assets/
Disallow: /css/
Disallow: /img/
Disallow: /js/
Disallow: /prelist-css/
Disallow: /prelist-img/
Disallow: /svgs/
Disallow: /vendors/

# disallow crawlers which obey the robots protocol
# User-agent: *
# Disallow: /