# ============================================================
#  robots.txt - newsvib.ru
# ============================================================

User-agent: *
Allow: /

# Service folders
Disallow: /admin/
Disallow: /includes/
Disallow: /ajax/
Disallow: /cgi-bin/

# Service files
Disallow: /login.php
Disallow: /register.php
Disallow: /logout.php
Disallow: /profile.php
Disallow: /daily-bonus.php
Disallow: /advertiser.php

# Duplicates and query params
Disallow: /*?page=1
Disallow: /*?sort=
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?fbclid=
Disallow: /*?payment_id=
Disallow: /*?success=
Disallow: /*?error=
Disallow: /*&

# Shop files (no index)
Disallow: /uploads/shop/

# Allow important
Allow: /category/
Allow: /tag/
Allow: /news/
Allow: /article/
Allow: /articles
Allow: /clip/
Allow: /clips
Allow: /video/
Allow: /shop
Allow: /shop/item/
Allow: /uploads/articles/
Allow: /uploads/news/
Allow: /assets/

# Sitemap
Sitemap: https://newsvib.ru/sitemap.xml
Sitemap: https://newsvib.ru/sitemap.php

# Yandex
User-agent: Yandex
Allow: /
Disallow: /admin/
Disallow: /includes/
Disallow: /ajax/
Disallow: /*?page=1
Disallow: /*?sort=
Disallow: /*?utm_
Disallow: /*?ref=
Disallow: /*?fbclid=
Clean-param: utm_source&utm_medium&utm_campaign&utm_term&utm_content&ref&fbclid
Crawl-delay: 1

# Google
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /includes/
Disallow: /ajax/
Disallow: /*?page=1
Disallow: /*?sort=

# Bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /