From e3fc0a0d7b650db6304011fe6e813e22b93f03c9 Mon Sep 17 00:00:00 2001
From: Nemo <commits@captnemo.in>
Date: Tue, 11 Jan 2022 20:45:51 +0530
Subject: [PATCH] New script
---
.gitignore | 8 ++++++++
.travis.yml | 22 ++++++++++------------
README.md | 2 +-
footer.md | 6 ++++++
index.md | 6 ++++++
requirements.txt | 1 +
scan.sh | 32 +++++++++++++++++++++-----------
script.py | 43 +++++++++++++++++++++++++++++++++++++++++++
template.md | 6 ------
website/Gemfile.lock | 62 ++++++++++++++++++++++++++++++++++++++++++--------------------
10 files changed, 124 insertions(+), 64 deletions(-)
diff --git a/.gitignore b/.gitignore
index 6787d47..122b618 100644
--- a/.gitignore
+++ a/.gitignore
@@ -1,2 +1,10 @@
tlds.txt
deploy-key
+
+tld/*.txt
+whois/*.txt
+.idea/
+__pypackages__
+.pdm.toml
+website/root.zone.txt
+template.mddiff --git a/.travis.yml b/.travis.yml
index 459cda2..23bdbd8 100644
--- a/.travis.yml
+++ a/.travis.yml
@@ -1,16 +1,17 @@
language: ruby
env:
global:
- - GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
+ - GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
addons:
apt:
packages:
- # required to avoid SSL errors
- - libcurl4-openssl-dev
- - dnsutils
- - idn
+ # required to avoid SSL errors
+ - libcurl4-openssl-dev
+ - dnsutils
+ - idn
+ - python3-dnspython
rvm:
-- 2.6.3
+ - 3.0
before_install:
- openssl aes-256-cbc -K $encrypted_9e883639804e_key -iv $encrypted_9e883639804e_iv -in deploy-key.enc -out deploy-key -d
- chmod 600 deploy-key
@@ -22,15 +23,10 @@
- gem update --system
- gem install bundler
script:
- - wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document tlds.txt
- ./scan.sh
- - curl 'https://ipapi.co/yaml/' > website/_data/ip.json
- - echo "This scan was last run on $(date)" >> template.md
- - cp tlds.txt website/
- - cp template.md website/index.md
- ./ci.sh
# branch whitelist, only for GitHub Pages
branches:
except:
- # gh-pages is automatically committed
- - gh-pages+ # gh-pages is automatically committed
+ - gh-pages
diff --git a/README.md b/README.md
index 7eec52d..a44b214 100644
--- a/README.md
+++ a/README.md
@@ -1,4 +1,4 @@
-# tld-a-record
+# tld-a-record [](https://pdm.fming.dev)
Build scripts that maintain https://captnemo.in/tld-a-record/, which is a list of all TLDs with A records. Builds once-every-night.
diff --git a/footer.md b/footer.md
new file mode 100644
index 0000000..6941230 100644
--- /dev/null
+++ a/footer.md
@@ -1,0 +1,6 @@
+As of the time of scan:
+
+- [List of all TLDs](tlds.txt) (source: <https://data.iana.org/TLD/tlds-alpha-by-domain.txt>)
+- [root.zone file](root.zone.txt) (source: <https://www.internic.net/domain/root.zone>)
+
+
diff --git a/index.md b/index.md
new file mode 100644
index 0000000..dcba2e9 100644
--- /dev/null
+++ a/index.md
@@ -1,0 +1,6 @@
+---
+layout: home
+---
+
+domain|punycode|whois|records
+------|--------|----|-----
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..90ec607 100644
--- /dev/null
+++ a/requirements.txt
@@ -1,0 +1,1 @@
+dnspython==2.1.0
diff --git a/scan.sh b/scan.sh
index 2591287..dd09360 100755
--- a/scan.sh
+++ a/scan.sh
@@ -7,20 +7,24 @@
# the scan was run (might be relevant for DNS lookups) into
# `website/_data/ip.json`. Structure is at ipapi.co
-for domain in $(grep -v '^#' tlds.txt); do
- RESULT=$(dig +time=1 +tries=1 +short "$domain." | head -c -1 | tr '\n' '@' | sed 's/@/`,`/g' | grep -v "connection timed out")
- if [ ! -z "$RESULT" ]; then
- echo $domain
- DOMAIN_REAL="$domain"
- # Very crude regex for punycode domains
- if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then
- DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
- fi
- echo "|$DOMAIN_REAL|$domain|[http](http://$domain)|[https](https://$domain)|\`$RESULT\`|" >> template.md
+# A complete text of each TLD is kept inside "tld/$domain.txt"
+
+wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document website/tlds.txt
+wget 'https://ipapi.co/yaml/' --output-document website/_data/ip.json
+wget https://www.internic.net/domain/root.zone --output-document website/root.zone.txt
+mkdir tld whois
+
+for domain in $(grep -v '^#' website/tlds.txt); do
+ DOMAIN_REAL="$domain"
+ # Very crude regex for punycode domains
+ if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then
+ DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
fi
+ python script.py "$domain." | sort > "tld/$domain.txt"
+ whois "$domain." > "whois/$domain.txt"
+ echo "$DOMAIN_REAL|$domain|[whois](whois/$domain.txt)|[dns](tld/$domain.txt)" >> template.md
done
-
-curl 'https://ipapi.co/yaml/' > website/_data/ip.json
-echo >> template.md
-echo "This scan was last run on $(date)" >> template.md
+echo "This scan was last run on $(date)" >> footer.md
+cat index.md footer.md > website/index.md
+cp -r whois tld website/diff --git a/script.py b/script.py
new file mode 100644
index 0000000..38b69c0 100644
--- /dev/null
+++ a/script.py
@@ -1,0 +1,43 @@
+import dns.resolver
+import sys
+
+# Make sure you have dnspython installed
+
+name = sys.argv[1]
+for qtype in [
+ "A",
+ "AAAA",
+ "APL",
+ "CAA",
+ "CDNSKEY",
+ "CDS",
+ "CERT",
+ "CNAME",
+ "CSYNC",
+ "DLV",
+ "DNAME",
+ "DNSKEY",
+ "DS",
+ "HTTPS",
+ "IPSECKEY",
+ "KEY",
+ "LOC",
+ "MX",
+ "NS",
+ "OPENPGPKEY",
+ "PTR",
+ "RP",
+ "RRSIG",
+ "SMIMEA",
+ "SOA",
+ "SRV",
+ "SSHFP",
+ "SVCB",
+ "TA",
+ "TLSA",
+ "TXT",
+ "URI",
+]:
+ answer = dns.resolver.resolve(name, qtype, raise_on_no_answer=False)
+ if answer.rrset is not None:
+ print(answer.rrset)
diff --git a/template.md b/template.md
deleted file mode 100644
index 83b545d..0000000 100644
--- a/template.md
+++ /dev/null
@@ -1,6 +1,0 @@
----
-layout: home
----
-
-|domain|punycode|http|https|lookup|
-|------|--------|----|-----|------|
diff --git a/website/Gemfile.lock b/website/Gemfile.lock
index 40244ea..fdf4cd4 100644
--- a/website/Gemfile.lock
+++ a/website/Gemfile.lock
@@ -1,38 +1,40 @@
GEM
remote: https://rubygems.org/
specs:
- addressable (2.7.0)
+ addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
colorator (1.1.0)
- concurrent-ruby (1.1.7)
- em-websocket (0.5.1)
+ concurrent-ruby (1.1.9)
+ em-websocket (0.5.3)
eventmachine (>= 0.12.9)
- http_parser.rb (~> 0.6.0)
+ http_parser.rb (~> 0)
eventmachine (1.2.7)
- ffi (1.13.1)
+ ffi (1.15.5)
forwardable-extended (2.6.0)
- http_parser.rb (0.6.0)
- i18n (0.9.5)
+ http_parser.rb (0.8.0)
+ i18n (1.8.11)
concurrent-ruby (~> 1.0)
- jekyll (3.9.0)
+ jekyll (4.2.1)
addressable (~> 2.4)
colorator (~> 1.0)
em-websocket (~> 0.5)
- i18n (~> 0.7)
- jekyll-sass-converter (~> 1.0)
+ i18n (~> 1.0)
+ jekyll-sass-converter (~> 2.0)
jekyll-watch (~> 2.0)
- kramdown (>= 1.17, < 3)
+ kramdown (~> 2.3)
+ kramdown-parser-gfm (~> 1.0)
liquid (~> 4.0)
- mercenary (~> 0.3.3)
+ mercenary (~> 0.4.0)
pathutil (~> 0.9)
- rouge (>= 1.7, < 4)
+ rouge (~> 3.0)
safe_yaml (~> 1.0)
- jekyll-sass-converter (1.5.2)
- sass (~> 3.4)
- jekyll-seo-tag (2.6.1)
- jekyll (>= 3.3, < 5.0)
- jekyll-theme-dinky (0.1.1)
- jekyll (~> 3.5)
+ terminal-table (~> 2.0)
+ jekyll-sass-converter (2.1.0)
+ sassc (> 2.0.1, < 3.0)
+ jekyll-seo-tag (2.7.1)
+ jekyll (>= 3.8, < 5.0)
+ jekyll-theme-dinky (0.2.0)
+ jekyll (> 3.5, < 5.0)
jekyll-seo-tag (~> 2.0)
jekyll-watch (2.2.1)
listen (~> 3.0)
@@ -41,24 +43,24 @@
kramdown-parser-gfm (1.1.0)
kramdown (~> 2.0)
liquid (4.0.3)
- listen (3.2.1)
+ listen (3.7.0)
rb-fsevent (~> 0.10, >= 0.10.3)
rb-inotify (~> 0.9, >= 0.9.10)
- mercenary (0.3.6)
+ mercenary (0.4.0)
pathutil (0.16.2)
forwardable-extended (~> 2.6)
- public_suffix (4.0.5)
- rb-fsevent (0.10.4)
+ public_suffix (4.0.6)
+ rb-fsevent (0.11.0)
rb-inotify (0.10.1)
ffi (~> 1.0)
rexml (3.2.5)
- rouge (3.22.0)
+ rouge (3.27.0)
safe_yaml (1.0.5)
- sass (3.7.4)
- sass-listen (~> 4.0.0)
- sass-listen (4.0.0)
- rb-fsevent (~> 0.9, >= 0.9.4)
- rb-inotify (~> 0.9, >= 0.9.7)
+ sassc (2.4.0)
+ ffi (~> 1.9)
+ terminal-table (2.0.0)
+ unicode-display_width (~> 1.1, >= 1.1.1)
+ unicode-display_width (1.8.0)
PLATFORMS
ruby
@@ -69,4 +71,4 @@
kramdown-parser-gfm
BUNDLED WITH
- 2.1.4
+ 2.2.26
--
rgit 0.1.5