[ci skip] Generate quoted CSVs
This commit is contained in:
parent
f3f21f1e0c
commit
ce9e5ab4c7
13
src/fetch.sh
13
src/fetch.sh
|
@ -24,11 +24,11 @@ function fetch_page() {
|
||||||
--data cnum=$1 \
|
--data cnum=$1 \
|
||||||
--data "page_no=$2" | \
|
--data "page_no=$2" | \
|
||||||
$PUP_BINARY '#nsdl-tables tr json{}' | \
|
$PUP_BINARY '#nsdl-tables tr json{}' | \
|
||||||
# generate 6 lines (second column has a link, so parse that) with raw output
|
# Generate a CSV (this contains the header row as well)
|
||||||
jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|.[]' | \
|
jq --raw-output '.[] | [.children[1].children[0].text, .children[2].text, .children[3].text,.children[4].text,.children[5].text]|@csv' | \
|
||||||
# and create a CSV from every 5 lines
|
# Convert & to &
|
||||||
paste -d, - - - - - | \
|
sed 's/&/\&/g' | \
|
||||||
# and we don't need the first row
|
# Drop the first row
|
||||||
tail -n +2 >> "$3"
|
tail -n +2 >> "$3"
|
||||||
}
|
}
|
||||||
function fetch_total_pages() {
|
function fetch_total_pages() {
|
||||||
|
@ -61,4 +61,5 @@ sem --wait
|
||||||
sort -o "$CLASS.csv" "$CLASS.csv"
|
sort -o "$CLASS.csv" "$CLASS.csv"
|
||||||
# Remove lines that don't start with the correct prefix
|
# Remove lines that don't start with the correct prefix
|
||||||
# This is to avoid ISINs like INF955L01IN9 showing up under IN9
|
# This is to avoid ISINs like INF955L01IN9 showing up under IN9
|
||||||
sed -i "/^$CLASS/!d" "$CLASS.csv"
|
# Note that there is a " at the beginning to account for quoted CSVs
|
||||||
|
sed -i "/^\"$CLASS/!d" "$CLASS.csv"
|
||||||
|
|
Loading…
Reference in New Issue