2019-04-27 11:03:52 +00:00
|
|
|
|
User-agent: MJ12bot
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SemrushBot
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SemrushBot-SA
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: rogerbot
|
|
|
|
|
|
Disallow:/
|
|
|
|
|
|
User-agent: dotbot
|
|
|
|
|
|
Disallow:/
|
|
|
|
|
|
User-agent: AhrefsBot
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Alexibot
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: SurveyBot
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Xenu’s
|
|
|
|
|
|
Disallow: /
|
|
|
|
|
|
User-agent: Xenu’s Link Sleuth 1.1c
|
|
|
|
|
|
Disallow: /
|
2023-06-01 08:30:49 +00:00
|
|
|
|
User-agent: AhrefsBot
|
|
|
|
|
|
Disallow: /
|
