New script
Diff
.gitignore | 8 ++++++++
.travis.yml | 22 ++++++++++------------
README.md | 2 +-
footer.md | 6 ++++++
index.md | 6 ++++++
requirements.txt | 1 +
scan.sh | 32 +++++++++++++++++++++-----------
script.py | 43 +++++++++++++++++++++++++++++++++++++++++++
template.md | 6 ------
website/Gemfile.lock | 62 ++++++++++++++++++++++++++++++++++++++++++--------------------
10 files changed, 124 insertions(+), 64 deletions(-)
@@ -1,2 +1,10 @@
tlds.txt
deploy-key
tld/*.txt
whois/*.txt
.idea/
__pypackages__
.pdm.toml
website/root.zone.txt
template.md
@@ -1,16 +1,17 @@
language: ruby
env:
global:
- GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
- GIT_REMOTE: git@github.com:captn3m0/tld-a-record.git
addons:
apt:
packages:
- libcurl4-openssl-dev
- dnsutils
- idn
- libcurl4-openssl-dev
- dnsutils
- idn
- python3-dnspython
rvm:
- 2.6.3
- 3.0
before_install:
- openssl aes-256-cbc -K $encrypted_9e883639804e_key -iv $encrypted_9e883639804e_iv -in deploy-key.enc -out deploy-key -d
- chmod 600 deploy-key
@@ -22,15 +23,10 @@
- gem update --system
- gem install bundler
script:
- wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document tlds.txt
- ./scan.sh
- curl 'https://ipapi.co/yaml/' > website/_data/ip.json
- echo "This scan was last run on $(date)" >> template.md
- cp tlds.txt website/
- cp template.md website/index.md
- ./ci.sh
branches:
except:
- gh-pages
- gh-pages
@@ -1,4 +1,4 @@
# tld-a-record
# tld-a-record [](https://pdm.fming.dev)
Build scripts that maintain https://captnemo.in/tld-a-record/, which is a list of all TLDs with A records. Builds once-every-night.
@@ -1,0 +1,6 @@
As of the time of scan:
- [List of all TLDs](tlds.txt) (source: <https://data.iana.org/TLD/tlds-alpha-by-domain.txt>)
- [root.zone file](root.zone.txt) (source: <https://www.internic.net/domain/root.zone>)
@@ -1,0 +1,6 @@
---
layout: home
---
domain|punycode|whois|records
------|--------|----|-----
@@ -1,0 +1,1 @@
dnspython==2.1.0
@@ -7,20 +7,24 @@
for domain in $(grep -v '^#' tlds.txt); do
RESULT=$(dig +time=1 +tries=1 +short "$domain." | head -c -1 | tr '\n' '@' | sed 's/@/`,`/g' | grep -v "connection timed out")
if [ ! -z "$RESULT" ]; then
echo $domain
DOMAIN_REAL="$domain"
if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then
DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
fi
echo "|$DOMAIN_REAL|$domain|[http](http://$domain)|[https](https://$domain)|\`$RESULT\`|" >> template.md
wget "https://data.iana.org/TLD/tlds-alpha-by-domain.txt" --output-document website/tlds.txt
wget 'https://ipapi.co/yaml/' --output-document website/_data/ip.json
wget https://www.internic.net/domain/root.zone --output-document website/root.zone.txt
mkdir tld whois
for domain in $(grep -v '^#' website/tlds.txt); do
DOMAIN_REAL="$domain"
if [[ $(echo "$domain" | grep -E "^XN--[[:upper:]]+$") ]]; then
DOMAIN_REAL=$(idn --idna-to-unicode "$domain")
fi
python script.py "$domain." | sort > "tld/$domain.txt"
whois "$domain." > "whois/$domain.txt"
echo "$DOMAIN_REAL|$domain|[whois](whois/$domain.txt)|[dns](tld/$domain.txt)" >> template.md
done
curl 'https://ipapi.co/yaml/' > website/_data/ip.json
echo >> template.md
echo "This scan was last run on $(date)" >> template.md
echo "This scan was last run on $(date)" >> footer.md
cat index.md footer.md > website/index.md
cp -r whois tld website/
@@ -1,0 +1,43 @@
import dns.resolver
import sys
name = sys.argv[1]
for qtype in [
"A",
"AAAA",
"APL",
"CAA",
"CDNSKEY",
"CDS",
"CERT",
"CNAME",
"CSYNC",
"DLV",
"DNAME",
"DNSKEY",
"DS",
"HTTPS",
"IPSECKEY",
"KEY",
"LOC",
"MX",
"NS",
"OPENPGPKEY",
"PTR",
"RP",
"RRSIG",
"SMIMEA",
"SOA",
"SRV",
"SSHFP",
"SVCB",
"TA",
"TLSA",
"TXT",
"URI",
]:
answer = dns.resolver.resolve(name, qtype, raise_on_no_answer=False)
if answer.rrset is not None:
print(answer.rrset)
@@ -1,6 +1,0 @@
---
layout: home
---
|domain|punycode|http|https|lookup|
|------|--------|----|-----|------|
@@ -1,38 +1,40 @@
GEM
remote: https://rubygems.org/
specs:
addressable (2.7.0)
addressable (2.8.0)
public_suffix (>= 2.0.2, < 5.0)
colorator (1.1.0)
concurrent-ruby (1.1.7)
em-websocket (0.5.1)
concurrent-ruby (1.1.9)
em-websocket (0.5.3)
eventmachine (>= 0.12.9)
http_parser.rb (~> 0.6.0)
http_parser.rb (~> 0)
eventmachine (1.2.7)
ffi (1.13.1)
ffi (1.15.5)
forwardable-extended (2.6.0)
http_parser.rb (0.6.0)
i18n (0.9.5)
http_parser.rb (0.8.0)
i18n (1.8.11)
concurrent-ruby (~> 1.0)
jekyll (3.9.0)
jekyll (4.2.1)
addressable (~> 2.4)
colorator (~> 1.0)
em-websocket (~> 0.5)
i18n (~> 0.7)
jekyll-sass-converter (~> 1.0)
i18n (~> 1.0)
jekyll-sass-converter (~> 2.0)
jekyll-watch (~> 2.0)
kramdown (>= 1.17, < 3)
kramdown (~> 2.3)
kramdown-parser-gfm (~> 1.0)
liquid (~> 4.0)
mercenary (~> 0.3.3)
mercenary (~> 0.4.0)
pathutil (~> 0.9)
rouge (>= 1.7, < 4)
rouge (~> 3.0)
safe_yaml (~> 1.0)
jekyll-sass-converter (1.5.2)
sass (~> 3.4)
jekyll-seo-tag (2.6.1)
jekyll (>= 3.3, < 5.0)
jekyll-theme-dinky (0.1.1)
jekyll (~> 3.5)
terminal-table (~> 2.0)
jekyll-sass-converter (2.1.0)
sassc (> 2.0.1, < 3.0)
jekyll-seo-tag (2.7.1)
jekyll (>= 3.8, < 5.0)
jekyll-theme-dinky (0.2.0)
jekyll (> 3.5, < 5.0)
jekyll-seo-tag (~> 2.0)
jekyll-watch (2.2.1)
listen (~> 3.0)
@@ -41,24 +43,24 @@
kramdown-parser-gfm (1.1.0)
kramdown (~> 2.0)
liquid (4.0.3)
listen (3.2.1)
listen (3.7.0)
rb-fsevent (~> 0.10, >= 0.10.3)
rb-inotify (~> 0.9, >= 0.9.10)
mercenary (0.3.6)
mercenary (0.4.0)
pathutil (0.16.2)
forwardable-extended (~> 2.6)
public_suffix (4.0.5)
rb-fsevent (0.10.4)
public_suffix (4.0.6)
rb-fsevent (0.11.0)
rb-inotify (0.10.1)
ffi (~> 1.0)
rexml (3.2.5)
rouge (3.22.0)
rouge (3.27.0)
safe_yaml (1.0.5)
sass (3.7.4)
sass-listen (~> 4.0.0)
sass-listen (4.0.0)
rb-fsevent (~> 0.9, >= 0.9.4)
rb-inotify (~> 0.9, >= 0.9.7)
sassc (2.4.0)
ffi (~> 1.9)
terminal-table (2.0.0)
unicode-display_width (~> 1.1, >= 1.1.1)
unicode-display_width (1.8.0)
PLATFORMS
ruby
@@ -69,4 +71,4 @@
kramdown-parser-gfm
BUNDLED WITH
2.1.4
2.2.26