nemo
/
nebula
1
0
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Block robots from scraping git.captnemo.in 

- Too many requests
This commit is contained in:
Nemo 2019-04-27 16:33:52 +05:30
parent 3bc9e40b61
commit d748e65a37
3 changed files with 44 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
gitea/conf/public/robots.txt Normal file
View File

@ -0,0 +1,20 @@
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: rogerbot
Disallow:/
User-agent: dotbot
Disallow:/
User-agent: AhrefsBot
Disallow: /
User-agent: Alexibot
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: Xenu’s
Disallow: /
User-agent: Xenu’s Link Sleuth 1.1c
Disallow: /

20
gitea/conf/robots.txt Normal file
View File

@ -0,0 +1,20 @@
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: SemrushBot-SA
Disallow: /
User-agent: rogerbot
Disallow:/
User-agent: dotbot
Disallow:/
User-agent: AhrefsBot
Disallow: /
User-agent: Alexibot
Disallow: /
User-agent: SurveyBot
Disallow: /
User-agent: Xenu’s
Disallow: /
User-agent: Xenu’s Link Sleuth 1.1c
Disallow: /

4
gitea/main.tf
View File

@ -46,6 +46,10 @@ resource "docker_container" "gitea" {
content = "${file("${path.module}/../docker/conf/humans.txt")}"
file = "/data/gitea/public/humans.txt"
}
upload {
content = "${file("${path.module}/conf/public/robots.txt")}"
file = "/data/gitea/public/robots.txt"
}
# Extra Links in header
upload {
content = "${file("${path.module}/conf/extra_links.tmpl")}"