Updates
This commit is contained in:
parent
d64df33c1f
commit
4a8bd2dd2b
|
@ -1,16 +1,32 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
doit() {
|
||||||
|
ID=$1
|
||||||
|
curl -s "http://goidirectory.nic.in/sitecounter.php?id=$ID" |grep '<meta http-equiv="refresh"' >> urls.raw.txt
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f doit
|
||||||
|
|
||||||
# Download from the goidirectory website
|
# Download from the goidirectory website
|
||||||
|
echo "Downloading from the GOI Directory"
|
||||||
for i in $(seq 1 14648); do
|
for i in $(seq 1 14648); do
|
||||||
echo $i
|
sem -j30 doit $i
|
||||||
curl -s "http://goidirectory.nic.in/sitecounter.php?id=$i" |grep '<meta http-equiv="refresh"' >> urls.raw.txt
|
|
||||||
done
|
done
|
||||||
|
|
||||||
|
sem --wait
|
||||||
|
|
||||||
# Get crts issued in gov.in
|
# Get crts issued in gov.in
|
||||||
# TODO: This isn't as good as the censys.io export
|
# TODO: This isn't as good as the censys.io export
|
||||||
# that pulse uses, switch to that
|
# that pulse uses, switch to that
|
||||||
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in';" | psql -h crt.sh -p 5432 -U guest certwatch > crt.sh.domains.txt
|
echo "Downloading from the crt.sh database"
|
||||||
|
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000;" | psql -h crt.sh -p 5432 -U guest certwatch > crt.sh.domains.txt
|
||||||
|
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 1000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
|
||||||
|
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 2000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
|
||||||
|
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 3000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
|
||||||
|
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 4000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
|
||||||
|
echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in' LIMIT 1000 OFFSET 5000;" | psql -h crt.sh -p 5432 -U guest certwatch >> crt.sh.domains.txt
|
||||||
|
|
||||||
|
echo "Combining them together"
|
||||||
cat urls.raw.txt crt.sh.domains.txt |sort -u > combined.txt
|
cat urls.raw.txt crt.sh.domains.txt |sort -u > combined.txt
|
||||||
|
|
||||||
php parse.php | sort -u > domains.csv
|
php parse.php | sort -u > domains.csv
|
1380
domains.csv
1380
domains.csv
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue