New script

.gitignore

@@ -1,2 +1,10 @@
 tlds.txt
 deploy-key
+
+tld/*.txt
+whois/*.txt
+.idea/
+__pypackages__
+.pdm.toml
+website/root.zone.txt
+template.md

.travis.yml

@@ -1,16 +1,17 @@
 language: ruby
 env:
   global:
-  - GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
+    - GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
 addons:
   apt:
     packages:
-    # required to avoid SSL errors
+      # required to avoid SSL errors
-    - libcurl4-openssl-dev
+      - libcurl4-openssl-dev
-    - dnsutils
+      - dnsutils
-    - idn
+      - idn
+      - python3-dnspython
 rvm:
-- 2.6.3
+  - 3.0
 before_install:
   - openssl aes-256-cbc -K $encrypted_9e883639804e_key -iv $encrypted_9e883639804e_iv -in deploy-key.enc -out deploy-key -d
   - chmod 600 deploy-key
@@ -22,15 +23,10 @@ before_install:
   - gem update --system
   - gem install bundler
 script:
-  - wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document tlds.txt
   - ./scan.sh
-  - curl 'https://ipapi.co/yaml/' > website/_data/ip.json
-  - echo "This scan was last run on $(date)" >> template.md
-  - cp tlds.txt website/
-  - cp template.md website/index.md
   - ./ci.sh
 # branch whitelist, only for GitHub Pages
 branches:
   except:
-  # gh-pages is automatically committed
+    # gh-pages is automatically committed
-  - gh-pages
+    - gh-pages

README.md

@@ -1,4 +1,4 @@
-# tld-a-record
+# tld-a-record [![pdm-managed](https://img.shields.io/badge/pdm-managed-blueviolet)](https://pdm.fming.dev)
 
 Build scripts that maintain https://captnemo.in/tld-a-record/, which is a list of all TLDs with A records. Builds once-every-night.
 

footer.md

@@ -0,0 +1,6 @@
+As of the time of scan:
+
+- [List of all TLDs](tlds.txt) (source: <https://data.iana.org/TLD/tlds-alpha-by-domain.txt>)
+- [root.zone file](root.zone.txt) (source: <https://www.internic.net/domain/root.zone>)
+
+

index.md

@@ -0,0 +1,6 @@
+---
+layout: home
+---
+
+domain|punycode|whois|records
+------|--------|----|-----

requirements.txt

@@ -0,0 +1 @@
+dnspython==2.1.0

scan.sh

@@ -7,20 +7,24 @@
 # the scan was run (might be relevant for DNS lookups) into
 # `website/_data/ip.json`. Structure is at ipapi.co
 
-for domain in $(grep -v '^#' tlds.txt); do
+# A complete text of each TLD is kept inside "tld/$domain.txt"
-    RESULT=$(dig +time=1 +tries=1 +short "$domain." | head -c -1 | tr '\n' '@' | sed 's/@/`,`/g' | grep -v "connection timed out")
+
-    if [ ! -z "$RESULT" ]; then
+wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document website/tlds.txt
-        echo $domain
+wget 'https://ipapi.co/yaml/' --output-document website/_data/ip.json
-        DOMAIN_REAL="$domain"
+wget https://www.internic.net/domain/root.zone --output-document website/root.zone.txt
-        # Very crude regex for punycode domains
+mkdir tld whois
-        if [[ $(echo "$domain" | grep -E  "^XN--[[:upper:]]+$") ]]; then
+
-            DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
+for domain in $(grep -v '^#' website/tlds.txt); do
-        fi
+    DOMAIN_REAL="$domain"
-        echo "|$DOMAIN_REAL|$domain|[http](http://$domain)|[https](https://$domain)|\`$RESULT\`|" >> template.md
+    # Very crude regex for punycode domains
+    if [[ $(echo "$domain" | grep -E  "^XN--[[:upper:]]+$") ]]; then
+        DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
     fi
+    python script.py "$domain." | sort > "tld/$domain.txt"
+    whois "$domain." > "whois/$domain.txt"
+    echo "$DOMAIN_REAL|$domain|[whois](whois/$domain.txt)|[dns](tld/$domain.txt)" >> template.md
 done
 
-curl 'https://ipapi.co/yaml/' > website/_data/ip.json
+echo "This scan was last run on $(date)" >> footer.md
-
+cat index.md footer.md > website/index.md
-echo >> template.md
+cp -r whois tld website/
-echo "This scan was last run on $(date)" >> template.md

script.py

@@ -0,0 +1,43 @@
+import dns.resolver
+import sys
+
+# Make sure you have dnspython installed
+
+name = sys.argv[1]
+for qtype in [
+    "A",
+    "AAAA",
+    "APL",
+    "CAA",
+    "CDNSKEY",
+    "CDS",
+    "CERT",
+    "CNAME",
+    "CSYNC",
+    "DLV",
+    "DNAME",
+    "DNSKEY",
+    "DS",
+    "HTTPS",
+    "IPSECKEY",
+    "KEY",
+    "LOC",
+    "MX",
+    "NS",
+    "OPENPGPKEY",
+    "PTR",
+    "RP",
+    "RRSIG",
+    "SMIMEA",
+    "SOA",
+    "SRV",
+    "SSHFP",
+    "SVCB",
+    "TA",
+    "TLSA",
+    "TXT",
+    "URI",
+]:
+    answer = dns.resolver.resolve(name, qtype, raise_on_no_answer=False)
+    if answer.rrset is not None:
+        print(answer.rrset)

template.md

@@ -1,6 +0,0 @@
----
-layout: home
----
-
-|domain|punycode|http|https|lookup|
-|------|--------|----|-----|------|

website/Gemfile.lock

@@ -1,38 +1,40 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
     colorator (1.1.0)
-    concurrent-ruby (1.1.7)
+    concurrent-ruby (1.1.9)
-    em-websocket (0.5.1)
+    em-websocket (0.5.3)
       eventmachine (>= 0.12.9)
-      http_parser.rb (~> 0.6.0)
+      http_parser.rb (~> 0)
     eventmachine (1.2.7)
-    ffi (1.13.1)
+    ffi (1.15.5)
     forwardable-extended (2.6.0)
-    http_parser.rb (0.6.0)
+    http_parser.rb (0.8.0)
-    i18n (0.9.5)
+    i18n (1.8.11)
       concurrent-ruby (~> 1.0)
-    jekyll (3.9.0)
+    jekyll (4.2.1)
       addressable (~> 2.4)
       colorator (~> 1.0)
       em-websocket (~> 0.5)
-      i18n (~> 0.7)
+      i18n (~> 1.0)
-      jekyll-sass-converter (~> 1.0)
+      jekyll-sass-converter (~> 2.0)
       jekyll-watch (~> 2.0)
-      kramdown (>= 1.17, < 3)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.0)
       liquid (~> 4.0)
-      mercenary (~> 0.3.3)
+      mercenary (~> 0.4.0)
       pathutil (~> 0.9)
-      rouge (>= 1.7, < 4)
+      rouge (~> 3.0)
       safe_yaml (~> 1.0)
-    jekyll-sass-converter (1.5.2)
+      terminal-table (~> 2.0)
-      sass (~> 3.4)
+    jekyll-sass-converter (2.1.0)
-    jekyll-seo-tag (2.6.1)
+      sassc (> 2.0.1, < 3.0)
-      jekyll (>= 3.3, < 5.0)
+    jekyll-seo-tag (2.7.1)
-    jekyll-theme-dinky (0.1.1)
+      jekyll (>= 3.8, < 5.0)
-      jekyll (~> 3.5)
+    jekyll-theme-dinky (0.2.0)
+      jekyll (> 3.5, < 5.0)
       jekyll-seo-tag (~> 2.0)
     jekyll-watch (2.2.1)
       listen (~> 3.0)
@@ -41,24 +43,24 @@ GEM
     kramdown-parser-gfm (1.1.0)
       kramdown (~> 2.0)
     liquid (4.0.3)
-    listen (3.2.1)
+    listen (3.7.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
-    mercenary (0.3.6)
+    mercenary (0.4.0)
     pathutil (0.16.2)
       forwardable-extended (~> 2.6)
-    public_suffix (4.0.5)
+    public_suffix (4.0.6)
-    rb-fsevent (0.10.4)
+    rb-fsevent (0.11.0)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     rexml (3.2.5)
-    rouge (3.22.0)
+    rouge (3.27.0)
     safe_yaml (1.0.5)
-    sass (3.7.4)
+    sassc (2.4.0)
-      sass-listen (~> 4.0.0)
+      ffi (~> 1.9)
-    sass-listen (4.0.0)
+    terminal-table (2.0.0)
-      rb-fsevent (~> 0.9, >= 0.9.4)
+      unicode-display_width (~> 1.1, >= 1.1.1)
-      rb-inotify (~> 0.9, >= 0.9.7)
+    unicode-display_width (1.8.0)
 
 PLATFORMS
   ruby
@@ -69,4 +71,4 @@ DEPENDENCIES
   kramdown-parser-gfm
 
 BUNDLED WITH
-   2.1.4
+   2.2.26