# Allow all bots to crawl
User-agent: *

# Disallow WordPress admin and core directories
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /wp-*.php

# Disallow specific WordPress files
Disallow: /wp-login.php
Disallow: /*.php$
Disallow: /feed/
Disallow: /comments/
Disallow: /author/
Disallow: /tag/
Disallow: /category/
Disallow: /*?
Disallow: */page

# Disallow specific file formats
Disallow: /*.js$    # JavaScript files
Disallow: /*.css$   # CSS files
Disallow: /*.pdf$   # PDF files
Disallow: /*.doc$   # Word documents
Disallow: /*.docx$  # Word documents
Disallow: /*.xls$   # Excel spreadsheets
Disallow: /*.xlsx$  # Excel spreadsheets
Disallow: /*.ppt$   # PowerPoint presentations
Disallow: /*.pptx$  # PowerPoint presentations
Disallow: /*.zip$   # ZIP archives
Disallow: /*.rar$   # RAR archives
Disallow: /*.gz$    # GZIP archives
Disallow: /*.exe$   # Executable files
Disallow: /*.tar$   # TAR archives
Disallow: /*.wmv$   # Windows Media Video files
Disallow: /*.avi$   # AVI video files
Disallow: /*.mov$   # QuickTime video files
Disallow: /*.mp3$   # MP3 audio files
Disallow: /*.mp4$   # MP4 video files
Disallow: /*.mpeg$  # MPEG video files
Disallow: /*.webm$  # WebM video files
Disallow: /*.ogg$   # OGG audio files
Disallow: /*.wav$   # WAV audio files

# Disallow crawling of specific file formats
Disallow: /*.php
Disallow: /*.php$
Disallow: /*.php?

# Exclude common WordPress file formats
Disallow: /*.php$
Disallow: /*.php?
Disallow: /*.css
Disallow: /*.js
Disallow: /*.json
Disallow: /*.txt
Disallow: /*.html
Disallow: /*.htm
Disallow: /*.png
Disallow: /*.jpg
Disallow: /*.jpeg
Disallow: /*.gif
Disallow: /*.bmp
Disallow: /*.ico
Disallow: /*.pdf

# Exclude backup files
Disallow: /*.bak$
Disallow: /*.backup$
Disallow: /*.sql$
Disallow: /*.zip$
Disallow: /*.tar$
Disallow: /*.gz$
Disallow: /*.rar$
Disallow: /*.tgz$
Disallow: /*.log$
Disallow: /*.tar.gz$
Disallow: /*.tar.bz2$

# Exclude temporary files
Disallow: /*.tmp$
Disallow: /*.temp$
Disallow: /*.temp$
Disallow: /*.swp$
Disallow: /*.bak$
Disallow: /*.~*$

# Exclude configuration files
Disallow: /wp-config.php
Disallow: /.htaccess

# Exclude readme files
Disallow: /readme.html

# Set a crawl delay of 5 seconds
Crawl-Delay: 5

# Allow XML Sitemap file
Allow: /page-sitemap.xml

# Specify the sitemap location
Sitemap: https://www.alltop9.com/sitemap_index.xml