Auto Updates FTW
This commit is contained in:
parent
e726415cb3
commit
decdbada40
|
@ -0,0 +1,22 @@
|
||||||
|
name: Update Data
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
# Run this on Sundays because NSDL won't add new things then
|
||||||
|
schedule:
|
||||||
|
# 8:07 UTC every Sunday
|
||||||
|
# 13:37 IST every Sunday
|
||||||
|
- cron: '7 8 * * 0'
|
||||||
|
jobs:
|
||||||
|
update:
|
||||||
|
name: Update data
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
with:
|
||||||
|
ref: ${{ github.head_ref }}
|
||||||
|
- name: Update data
|
||||||
|
run: ./fetch.sh
|
||||||
|
- uses: stefanzweifel/git-auto-commit-action@v4
|
||||||
|
with:
|
||||||
|
commit_message: Update ISIN Data
|
||||||
|
commit_author: 'github-actions[bot] <github-actions[bot]@users.noreply.github.com>'
|
|
@ -0,0 +1,2 @@
|
||||||
|
pup
|
||||||
|
pup.zip
|
|
@ -4,6 +4,8 @@ ISIN Data from various public securities.
|
||||||
|
|
||||||
Source: NSDL provides a ISIN Search at <https://nsdl.co.in/master_search.php>.
|
Source: NSDL provides a ISIN Search at <https://nsdl.co.in/master_search.php>.
|
||||||
|
|
||||||
|
Automatically updated every Sunday using GitHub Actions.
|
||||||
|
|
||||||
Currently tracked:
|
Currently tracked:
|
||||||
|
|
||||||
|File|Issuer|
|
|File|Issuer|
|
||||||
|
|
|
@ -1,13 +1,29 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
|
export PUP_BINARY="$(which pup)"
|
||||||
|
|
||||||
|
if ! command -v pup &> /dev/null
|
||||||
|
then
|
||||||
|
wget https://github.com/ericchiang/pup/releases/download/v0.4.0/pup_v0.4.0_linux_amd64.zip -O pup.zip
|
||||||
|
unzip pup.zip
|
||||||
|
rm pup.zip
|
||||||
|
chmod +x ./pup
|
||||||
|
export PUP_BINARY="$(pwd)/pup"
|
||||||
|
fi
|
||||||
|
|
||||||
# Call with INX Page_num file_name
|
# Call with INX Page_num file_name
|
||||||
function fetch_page() {
|
function fetch_page() {
|
||||||
echo "[+] $1/$2"
|
echo "[+] $1/$2"
|
||||||
curl "https://nsdl.co.in/master_search_res.php" \
|
curl "https://nsdl.co.in/master_search_res.php" \
|
||||||
|
--user-agent "Mozilla/Gecko/Firefox/58.0" \
|
||||||
--silent \
|
--silent \
|
||||||
|
--retry 3 \
|
||||||
|
--connect-timeout 10 \
|
||||||
|
--retry-max-time 30 \
|
||||||
--data cnum=$1 \
|
--data cnum=$1 \
|
||||||
--data "page_no=$2" |
|
--data "page_no=$2" |
|
||||||
# for each row
|
# for each row
|
||||||
pup '#nsdl-tables tr json{}' | \
|
$PUP_BINARY '#nsdl-tables tr json{}' | \
|
||||||
# generate 6 lines (second column has a link, so parse that) with raw output
|
# generate 6 lines (second column has a link, so parse that) with raw output
|
||||||
jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \
|
jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \
|
||||||
# and create a CSV from every 5 lines
|
# and create a CSV from every 5 lines
|
||||||
|
@ -17,10 +33,11 @@ function fetch_page() {
|
||||||
}
|
}
|
||||||
function fetch_total_pages() {
|
function fetch_total_pages() {
|
||||||
curl "https://nsdl.co.in/master_search_res.php" \
|
curl "https://nsdl.co.in/master_search_res.php" \
|
||||||
|
--user-agent "Mozilla/Gecko/Firefox/58.0" \
|
||||||
--silent \
|
--silent \
|
||||||
--data cnum=$1 \
|
--data cnum=$1 \
|
||||||
--data "page_no=1" |
|
--data "page_no=1" |
|
||||||
pup 'input[name=total_page] attr{value}'
|
$PUP_BINARY 'input[name=total_page] attr{value}'
|
||||||
}
|
}
|
||||||
export -f fetch_page
|
export -f fetch_page
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue