This commit is contained in:
Nemo 2019-01-14 16:20:06 +05:30
parent d64df33c1f
commit 4a8bd2dd2b
3 changed files with 1398 additions and 6 deletions

22
dl.sh Normal file → Executable file
View File

@ -1,16 +1,32 @@
#!/bin/bash #!/bin/bash
doit() {
ID=$1
curl -s "http://goidirectory.nic.in/sitecounter.php?id=$ID" |grep '<meta http-equiv="refresh"' >> urls.raw.txt
}
export -f doit
# Download from the goidirectory website # Download from the goidirectory website
echo "Downloading from the GOI Directory"
for i in $(seq 1 14648); do for i in $(seq 1 14648); do
echo $i sem -j30 doit $i
curl -s "http://goidirectory.nic.in/sitecounter.php?id=$i" |grep '<meta http-equiv="refresh"' >> urls.raw.txt
done done
sem --wait
# Get crts issued in gov.in # Get crts issued in gov.in
# TODO: This isn't as good as the censys.io export # TODO: This isn't as good as the censys.io export
# that pulse uses, switch to that # that pulse uses, switch to that
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in';" | psql -h crt.sh -p 5432 -U guest certwatch > crt.sh.domains.txt echo "Downloading from the crt.sh database"
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000;" | psql -h crt.sh -p 5432 -U guest certwatch > crt.sh.domains.txt
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 1000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 2000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 3000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 4000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 5000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
echo "Combining them together"
cat urls.raw.txt crt.sh.domains.txt |sort -u > combined.txt cat urls.raw.txt crt.sh.domains.txt |sort -u > combined.txt
php parse.php | sort -u > domains.csv php parse.php | sort -u > domains.csv

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
<?php <?php
foreach(file('combined.txt') as $row) foreach(file('combined.2.txt') as $row)
{ {
$row = trim($row); $row = trim($row);
$url = "http://$row/"; $url = "http://$row/";