# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders. Ensure all images, scripts,
# and important files are accessible by crawlers.

# Sitemap location
sitemap: https://bonitron.com/sitemap.xml

# Allow full access to important bots
User-agent: SiteAuditBot
Allow: /

User-agent: AhrefsBot
Allow: /


User-agent: Semrushbot-SI
Allow: /
  

User-agent: Googlebot
Allow: /

User-agent: *
# Allow specific file types globally
Allow: /*.css$
Allow: /*.js$
Allow: /*.gif$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.pdf$

# Allow crawling of /productimages/ directory
Allow: /Images/
Allow: /productimages/


# Disallow unnecessary directories and files
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/

# Disallow paths without clean URLs
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/

# Disallow crawling of STEP files
Disallow: /*.step$
# Prevent bots from crawling or indexing .STEP CAD files
User-agent: *
Disallow: /*.step$