diff --git a/.gitignore b/.gitignore index 6787d47..122b618 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,10 @@ tlds.txt deploy-key + +tld/*.txt +whois/*.txt +.idea/ +__pypackages__ +.pdm.toml +website/root.zone.txt +template.md \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 459cda2..23bdbd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,17 @@ language: ruby env: global: - - GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git + - GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git addons: apt: packages: - # required to avoid SSL errors - - libcurl4-openssl-dev - - dnsutils - - idn + # required to avoid SSL errors + - libcurl4-openssl-dev + - dnsutils + - idn + - python3-dnspython rvm: -- 2.6.3 + - 3.0 before_install: - openssl aes-256-cbc -K $encrypted_9e883639804e_key -iv $encrypted_9e883639804e_iv -in deploy-key.enc -out deploy-key -d - chmod 600 deploy-key @@ -22,15 +23,10 @@ before_install: - gem update --system - gem install bundler script: - - wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document tlds.txt - ./scan.sh - - curl 'https://ipapi.co/yaml/' > website/_data/ip.json - - echo "This scan was last run on $(date)" >> template.md - - cp tlds.txt website/ - - cp template.md website/index.md - ./ci.sh # branch whitelist, only for GitHub Pages branches: except: - # gh-pages is automatically committed - - gh-pages \ No newline at end of file + # gh-pages is automatically committed + - gh-pages diff --git a/README.md b/README.md index 7eec52d..a44b214 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# tld-a-record +# tld-a-record [![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev) Build scripts that maintain https://captnemo.in/tld-a-record/, which is a list of all TLDs with A records. Builds once-every-night. diff --git a/footer.md b/footer.md new file mode 100644 index 0000000..6941230 --- /dev/null +++ b/footer.md @@ -0,0 +1,6 @@ +As of the time of scan: + +- [List of all TLDs](tlds.txt) (source: ) +- [root.zone file](root.zone.txt) (source: ) + + diff --git a/index.md b/index.md new file mode 100644 index 0000000..dcba2e9 --- /dev/null +++ b/index.md @@ -0,0 +1,6 @@ +--- +layout: home +--- + +domain|punycode|whois|records +------|--------|----|----- diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..90ec607 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +dnspython==2.1.0 diff --git a/scan.sh b/scan.sh index 2591287..dd09360 100755 --- a/scan.sh +++ b/scan.sh @@ -7,20 +7,24 @@ # the scan was run (might be relevant for DNS lookups) into # `website/_data/ip.json`. Structure is at ipapi.co -for domain in $(grep -v '^#' tlds.txt); do - RESULT=$(dig +time=1 +tries=1 +short "$domain." | head -c -1 | tr '\n' '@' | sed 's/@/`,`/g' | grep -v "connection timed out") - if [ ! -z "$RESULT" ]; then - echo $domain - DOMAIN_REAL="$domain" - # Very crude regex for punycode domains - if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then - DOMAIN_REAL=$(idn --idna-to-unicode "$domain") - fi - echo "|$DOMAIN_REAL|$domain|[http](http://$domain)|[https](https://$domain)|\`$RESULT\`|" >> template.md +# A complete text of each TLD is kept inside "tld/$domain.txt" + +wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document website/tlds.txt +wget 'https://ipapi.co/yaml/' --output-document website/_data/ip.json +wget https://www.internic.net/domain/root.zone --output-document website/root.zone.txt +mkdir tld whois + +for domain in $(grep -v '^#' website/tlds.txt); do + DOMAIN_REAL="$domain" + # Very crude regex for punycode domains + if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then + DOMAIN_REAL=$(idn --idna-to-unicode "$domain") fi + python script.py "$domain." | sort > "tld/$domain.txt" + whois "$domain." > "whois/$domain.txt" + echo "$DOMAIN_REAL|$domain|[whois](whois/$domain.txt)|[dns](tld/$domain.txt)" >> template.md done -curl 'https://ipapi.co/yaml/' > website/_data/ip.json - -echo >> template.md -echo "This scan was last run on $(date)" >> template.md +echo "This scan was last run on $(date)" >> footer.md +cat index.md footer.md > website/index.md +cp -r whois tld website/ \ No newline at end of file diff --git a/script.py b/script.py new file mode 100644 index 0000000..38b69c0 --- /dev/null +++ b/script.py @@ -0,0 +1,43 @@ +import dns.resolver +import sys + +# Make sure you have dnspython installed + +name = sys.argv[1] +for qtype in [ + "A", + "AAAA", + "APL", + "CAA", + "CDNSKEY", + "CDS", + "CERT", + "CNAME", + "CSYNC", + "DLV", + "DNAME", + "DNSKEY", + "DS", + "HTTPS", + "IPSECKEY", + "KEY", + "LOC", + "MX", + "NS", + "OPENPGPKEY", + "PTR", + "RP", + "RRSIG", + "SMIMEA", + "SOA", + "SRV", + "SSHFP", + "SVCB", + "TA", + "TLSA", + "TXT", + "URI", +]: + answer = dns.resolver.resolve(name, qtype, raise_on_no_answer=False) + if answer.rrset is not None: + print(answer.rrset) diff --git a/template.md b/template.md deleted file mode 100644 index 83b545d..0000000 --- a/template.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -layout: home ---- - -|domain|punycode|http|https|lookup| -|------|--------|----|-----|------| diff --git a/website/Gemfile.lock b/website/Gemfile.lock index 40244ea..fdf4cd4 100644 --- a/website/Gemfile.lock +++ b/website/Gemfile.lock @@ -1,38 +1,40 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.7.0) + addressable (2.8.0) public_suffix (>= 2.0.2, < 5.0) colorator (1.1.0) - concurrent-ruby (1.1.7) - em-websocket (0.5.1) + concurrent-ruby (1.1.9) + em-websocket (0.5.3) eventmachine (>= 0.12.9) - http_parser.rb (~> 0.6.0) + http_parser.rb (~> 0) eventmachine (1.2.7) - ffi (1.13.1) + ffi (1.15.5) forwardable-extended (2.6.0) - http_parser.rb (0.6.0) - i18n (0.9.5) + http_parser.rb (0.8.0) + i18n (1.8.11) concurrent-ruby (~> 1.0) - jekyll (3.9.0) + jekyll (4.2.1) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) - i18n (~> 0.7) - jekyll-sass-converter (~> 1.0) + i18n (~> 1.0) + jekyll-sass-converter (~> 2.0) jekyll-watch (~> 2.0) - kramdown (>= 1.17, < 3) + kramdown (~> 2.3) + kramdown-parser-gfm (~> 1.0) liquid (~> 4.0) - mercenary (~> 0.3.3) + mercenary (~> 0.4.0) pathutil (~> 0.9) - rouge (>= 1.7, < 4) + rouge (~> 3.0) safe_yaml (~> 1.0) - jekyll-sass-converter (1.5.2) - sass (~> 3.4) - jekyll-seo-tag (2.6.1) - jekyll (>= 3.3, < 5.0) - jekyll-theme-dinky (0.1.1) - jekyll (~> 3.5) + terminal-table (~> 2.0) + jekyll-sass-converter (2.1.0) + sassc (> 2.0.1, < 3.0) + jekyll-seo-tag (2.7.1) + jekyll (>= 3.8, < 5.0) + jekyll-theme-dinky (0.2.0) + jekyll (> 3.5, < 5.0) jekyll-seo-tag (~> 2.0) jekyll-watch (2.2.1) listen (~> 3.0) @@ -41,24 +43,24 @@ GEM kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) liquid (4.0.3) - listen (3.2.1) + listen (3.7.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) - mercenary (0.3.6) + mercenary (0.4.0) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (4.0.5) - rb-fsevent (0.10.4) + public_suffix (4.0.6) + rb-fsevent (0.11.0) rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.5) - rouge (3.22.0) + rouge (3.27.0) safe_yaml (1.0.5) - sass (3.7.4) - sass-listen (~> 4.0.0) - sass-listen (4.0.0) - rb-fsevent (~> 0.9, >= 0.9.4) - rb-inotify (~> 0.9, >= 0.9.7) + sassc (2.4.0) + ffi (~> 1.9) + terminal-table (2.0.0) + unicode-display_width (~> 1.1, >= 1.1.1) + unicode-display_width (1.8.0) PLATFORMS ruby @@ -69,4 +71,4 @@ DEPENDENCIES kramdown-parser-gfm BUNDLED WITH - 2.1.4 + 2.2.26