# robots.txt for 开户网站
# https://kaihu-wap.club/

# 允许所有搜索引擎爬虫
User-agent: *
Allow: /

# 允许Yandex爬虫
User-agent: Yandex
Allow: /
Crawl-delay: 1

# 允许Bing爬虫
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# 允许Google爬虫
User-agent: Googlebot
Allow: /

# 允许百度爬虫
User-agent: Baiduspider
Allow: /
Crawl-delay: 1

# 禁止爬取的目录和文件
Disallow: /admin/
Disallow: /private/
Disallow: /*.pdf$
Disallow: /temp/

# 站点地图位置
Sitemap: https://kaihu-wap.club/sitemap.xml
Sitemap: https://kaihu-wap.club/sitemap.txt

# Host指令 (Yandex支持)
Host: https://kaihu-wap.club