New script
This commit is contained in:
parent
3cf6921331
commit
e3fc0a0d7b
|
@ -1,2 +1,10 @@
|
|||
tlds.txt
|
||||
deploy-key
|
||||
|
||||
tld/*.txt
|
||||
whois/*.txt
|
||||
.idea/
|
||||
__pypackages__
|
||||
.pdm.toml
|
||||
website/root.zone.txt
|
||||
template.md
|
22
.travis.yml
22
.travis.yml
|
@ -1,16 +1,17 @@
|
|||
language: ruby
|
||||
env:
|
||||
global:
|
||||
- GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
|
||||
- GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
# required to avoid SSL errors
|
||||
- libcurl4-openssl-dev
|
||||
- dnsutils
|
||||
- idn
|
||||
# required to avoid SSL errors
|
||||
- libcurl4-openssl-dev
|
||||
- dnsutils
|
||||
- idn
|
||||
- python3-dnspython
|
||||
rvm:
|
||||
- 2.6.3
|
||||
- 3.0
|
||||
before_install:
|
||||
- openssl aes-256-cbc -K $encrypted_9e883639804e_key -iv $encrypted_9e883639804e_iv -in deploy-key.enc -out deploy-key -d
|
||||
- chmod 600 deploy-key
|
||||
|
@ -22,15 +23,10 @@ before_install:
|
|||
- gem update --system
|
||||
- gem install bundler
|
||||
script:
|
||||
- wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document tlds.txt
|
||||
- ./scan.sh
|
||||
- curl 'https://ipapi.co/yaml/' > website/_data/ip.json
|
||||
- echo "This scan was last run on $(date)" >> template.md
|
||||
- cp tlds.txt website/
|
||||
- cp template.md website/index.md
|
||||
- ./ci.sh
|
||||
# branch whitelist, only for GitHub Pages
|
||||
branches:
|
||||
except:
|
||||
# gh-pages is automatically committed
|
||||
- gh-pages
|
||||
# gh-pages is automatically committed
|
||||
- gh-pages
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# tld-a-record
|
||||
# tld-a-record [![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev)
|
||||
|
||||
Build scripts that maintain https://captnemo.in/tld-a-record/, which is a list of all TLDs with A records. Builds once-every-night.
|
||||
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
As of the time of scan:
|
||||
|
||||
- [List of all TLDs](tlds.txt) (source: <https://data.iana.org/TLD/tlds-alpha-by-domain.txt>)
|
||||
- [root.zone file](root.zone.txt) (source: <https://www.internic.net/domain/root.zone>)
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
---
|
||||
layout: home
|
||||
---
|
||||
|
||||
domain|punycode|whois|records
|
||||
------|--------|----|-----
|
|
@ -0,0 +1 @@
|
|||
dnspython==2.1.0
|
32
scan.sh
32
scan.sh
|
@ -7,20 +7,24 @@
|
|||
# the scan was run (might be relevant for DNS lookups) into
|
||||
# `website/_data/ip.json`. Structure is at ipapi.co
|
||||
|
||||
for domain in $(grep -v '^#' tlds.txt); do
|
||||
RESULT=$(dig +time=1 +tries=1 +short "$domain." | head -c -1 | tr '\n' '@' | sed 's/@/`,`/g' | grep -v "connection timed out")
|
||||
if [ ! -z "$RESULT" ]; then
|
||||
echo $domain
|
||||
DOMAIN_REAL="$domain"
|
||||
# Very crude regex for punycode domains
|
||||
if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then
|
||||
DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
|
||||
fi
|
||||
echo "|$DOMAIN_REAL|$domain|[http](http://$domain)|[https](https://$domain)|\`$RESULT\`|" >> template.md
|
||||
# A complete text of each TLD is kept inside "tld/$domain.txt"
|
||||
|
||||
wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document website/tlds.txt
|
||||
wget 'https://ipapi.co/yaml/' --output-document website/_data/ip.json
|
||||
wget https://www.internic.net/domain/root.zone --output-document website/root.zone.txt
|
||||
mkdir tld whois
|
||||
|
||||
for domain in $(grep -v '^#' website/tlds.txt); do
|
||||
DOMAIN_REAL="$domain"
|
||||
# Very crude regex for punycode domains
|
||||
if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then
|
||||
DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
|
||||
fi
|
||||
python script.py "$domain." | sort > "tld/$domain.txt"
|
||||
whois "$domain." > "whois/$domain.txt"
|
||||
echo "$DOMAIN_REAL|$domain|[whois](whois/$domain.txt)|[dns](tld/$domain.txt)" >> template.md
|
||||
done
|
||||
|
||||
curl 'https://ipapi.co/yaml/' > website/_data/ip.json
|
||||
|
||||
echo >> template.md
|
||||
echo "This scan was last run on $(date)" >> template.md
|
||||
echo "This scan was last run on $(date)" >> footer.md
|
||||
cat index.md footer.md > website/index.md
|
||||
cp -r whois tld website/
|
|
@ -0,0 +1,43 @@
|
|||
import dns.resolver
|
||||
import sys
|
||||
|
||||
# Make sure you have dnspython installed
|
||||
|
||||
name = sys.argv[1]
|
||||
for qtype in [
|
||||
"A",
|
||||
"AAAA",
|
||||
"APL",
|
||||
"CAA",
|
||||
"CDNSKEY",
|
||||
"CDS",
|
||||
"CERT",
|
||||
"CNAME",
|
||||
"CSYNC",
|
||||
"DLV",
|
||||
"DNAME",
|
||||
"DNSKEY",
|
||||
"DS",
|
||||
"HTTPS",
|
||||
"IPSECKEY",
|
||||
"KEY",
|
||||
"LOC",
|
||||
"MX",
|
||||
"NS",
|
||||
"OPENPGPKEY",
|
||||
"PTR",
|
||||
"RP",
|
||||
"RRSIG",
|
||||
"SMIMEA",
|
||||
"SOA",
|
||||
"SRV",
|
||||
"SSHFP",
|
||||
"SVCB",
|
||||
"TA",
|
||||
"TLSA",
|
||||
"TXT",
|
||||
"URI",
|
||||
]:
|
||||
answer = dns.resolver.resolve(name, qtype, raise_on_no_answer=False)
|
||||
if answer.rrset is not None:
|
||||
print(answer.rrset)
|
|
@ -1,6 +0,0 @@
|
|||
---
|
||||
layout: home
|
||||
---
|
||||
|
||||
|domain|punycode|http|https|lookup|
|
||||
|------|--------|----|-----|------|
|
|
@ -1,38 +1,40 @@
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
addressable (2.7.0)
|
||||
addressable (2.8.0)
|
||||
public_suffix (>= 2.0.2, < 5.0)
|
||||
colorator (1.1.0)
|
||||
concurrent-ruby (1.1.7)
|
||||
em-websocket (0.5.1)
|
||||
concurrent-ruby (1.1.9)
|
||||
em-websocket (0.5.3)
|
||||
eventmachine (>= 0.12.9)
|
||||
http_parser.rb (~> 0.6.0)
|
||||
http_parser.rb (~> 0)
|
||||
eventmachine (1.2.7)
|
||||
ffi (1.13.1)
|
||||
ffi (1.15.5)
|
||||
forwardable-extended (2.6.0)
|
||||
http_parser.rb (0.6.0)
|
||||
i18n (0.9.5)
|
||||
http_parser.rb (0.8.0)
|
||||
i18n (1.8.11)
|
||||
concurrent-ruby (~> 1.0)
|
||||
jekyll (3.9.0)
|
||||
jekyll (4.2.1)
|
||||
addressable (~> 2.4)
|
||||
colorator (~> 1.0)
|
||||
em-websocket (~> 0.5)
|
||||
i18n (~> 0.7)
|
||||
jekyll-sass-converter (~> 1.0)
|
||||
i18n (~> 1.0)
|
||||
jekyll-sass-converter (~> 2.0)
|
||||
jekyll-watch (~> 2.0)
|
||||
kramdown (>= 1.17, < 3)
|
||||
kramdown (~> 2.3)
|
||||
kramdown-parser-gfm (~> 1.0)
|
||||
liquid (~> 4.0)
|
||||
mercenary (~> 0.3.3)
|
||||
mercenary (~> 0.4.0)
|
||||
pathutil (~> 0.9)
|
||||
rouge (>= 1.7, < 4)
|
||||
rouge (~> 3.0)
|
||||
safe_yaml (~> 1.0)
|
||||
jekyll-sass-converter (1.5.2)
|
||||
sass (~> 3.4)
|
||||
jekyll-seo-tag (2.6.1)
|
||||
jekyll (>= 3.3, < 5.0)
|
||||
jekyll-theme-dinky (0.1.1)
|
||||
jekyll (~> 3.5)
|
||||
terminal-table (~> 2.0)
|
||||
jekyll-sass-converter (2.1.0)
|
||||
sassc (> 2.0.1, < 3.0)
|
||||
jekyll-seo-tag (2.7.1)
|
||||
jekyll (>= 3.8, < 5.0)
|
||||
jekyll-theme-dinky (0.2.0)
|
||||
jekyll (> 3.5, < 5.0)
|
||||
jekyll-seo-tag (~> 2.0)
|
||||
jekyll-watch (2.2.1)
|
||||
listen (~> 3.0)
|
||||
|
@ -41,24 +43,24 @@ GEM
|
|||
kramdown-parser-gfm (1.1.0)
|
||||
kramdown (~> 2.0)
|
||||
liquid (4.0.3)
|
||||
listen (3.2.1)
|
||||
listen (3.7.0)
|
||||
rb-fsevent (~> 0.10, >= 0.10.3)
|
||||
rb-inotify (~> 0.9, >= 0.9.10)
|
||||
mercenary (0.3.6)
|
||||
mercenary (0.4.0)
|
||||
pathutil (0.16.2)
|
||||
forwardable-extended (~> 2.6)
|
||||
public_suffix (4.0.5)
|
||||
rb-fsevent (0.10.4)
|
||||
public_suffix (4.0.6)
|
||||
rb-fsevent (0.11.0)
|
||||
rb-inotify (0.10.1)
|
||||
ffi (~> 1.0)
|
||||
rexml (3.2.5)
|
||||
rouge (3.22.0)
|
||||
rouge (3.27.0)
|
||||
safe_yaml (1.0.5)
|
||||
sass (3.7.4)
|
||||
sass-listen (~> 4.0.0)
|
||||
sass-listen (4.0.0)
|
||||
rb-fsevent (~> 0.9, >= 0.9.4)
|
||||
rb-inotify (~> 0.9, >= 0.9.7)
|
||||
sassc (2.4.0)
|
||||
ffi (~> 1.9)
|
||||
terminal-table (2.0.0)
|
||||
unicode-display_width (~> 1.1, >= 1.1.1)
|
||||
unicode-display_width (1.8.0)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
@ -69,4 +71,4 @@ DEPENDENCIES
|
|||
kramdown-parser-gfm
|
||||
|
||||
BUNDLED WITH
|
||||
2.1.4
|
||||
2.2.26
|
||||
|
|
Loading…
Reference in New Issue