Single file release process

.github/workflows/update.yml

@@ -14,15 +14,15 @@ jobs:
       with:
         ref: ${{ github.head_ref }}
     - name: Update data
-      run: make all
+      run: make update
       id: update_data
     # Only tag if we're running on the scheduled job
     - uses: stefanzweifel/git-auto-commit-action@v4
       with:
         commit_message: Update ISIN Data
         commit_author: 'github-actions[bot] <github-actions[bot]@users.noreply.github.com>'
-        file_pattern: "IN*.csv"
-        status_options: '--untracked-files=normal'
+        file_pattern: "ISIN.csv"
+        status_options: '--untracked-files=no'
         tagging_message: "v${{ steps.update_data.outputs.version }}"
     - name: Create Release
       run: |

.gitignore

@@ -1,3 +1,4 @@
 pup
 release.md
 pup.zip
+IN*.csv

Makefile

@@ -1,32 +1,30 @@
 SHELL=/bin/bash
 version=`date +%Y.%-m.%-d`
 
-all: INE INF IN9 IN0 IN1 IN2 IN3 IN4 update
+# Build Process
+
+ISIN: INE INF IN9 IN0 IN1 IN2 IN3 IN4
+	cat header.csv IN*.csv > ISIN.csv
+	rm IN*.csv
 
 INE INF IN9 IN0 IN1 IN2 IN3 IN4:
 	./fetch.sh $@
-
-old:
-	git show HEAD^:INE.csv > /tmp/INE.csv
-	git show HEAD^:INF.csv > /tmp/INF.csv
-	git show HEAD^:IN9.csv > /tmp/IN9.csv
-	git show HEAD^:IN0.csv > /tmp/IN0.csv
-	git show HEAD^:IN1.csv > /tmp/IN1.csv
-	git show HEAD^:IN2.csv > /tmp/IN2.csv
-	git show HEAD^:IN3.csv > /tmp/IN3.csv
-	git show HEAD^:IN4.csv > /tmp/IN4.csv
-
-release.md: old
-	python3 diff.py > release.md
-
-release: release.md
-	gh release create "$(version)" --notes-file release.md IN*.csv
-
-update:
+update: ISIN
 	echo "::set-output name=version::$(version)"
 	sed -i "s/^version.*/version: $(version)/" CITATION.cff
 	sed -i "s/^date-released.*/date-released: `date --rfc-3339=date`/" CITATION.cff
 
 	jq ".version = \"$(version)\" | .created = \"`date --rfc-3339=seconds`\"" datapackage.json > d2.json
 	mv d2.json datapackage.json
-	git add CITATION.cff datapackage.json
+	git add CITATION.cff datapackage.json
+
+# Release Process
+
+old:
+	git show HEAD^:ISIN.csv > /tmp/ISIN.csv
+
+release.md: old
+	python3 diff.py > release.md
+
+release: release.md
+	gh release create "$(version)" --notes-file release.md ISIN.csv

diff.py

@@ -11,15 +11,10 @@ def chunks(lst, n):
     for i in range(0, len(lst), n):
         yield lst[i:i + n]
 
-for i in ['E', 'F', '9', '0', '1', '2', '3', '4']:
-    diff = compare(
-        load_csv(open("/tmp/IN%s.csv" % i), key="ISIN"),
-        load_csv(open("IN%s.csv" % i), key="ISIN"),
-        True
-    )
-    # print(diff)
-    added += diff['added']
-    changed += diff['changed']
-    removed += diff['removed']
+diff = compare(
+    load_csv(open("/tmp/ISIN.csv" % i), key="ISIN"),
+    load_csv(open("ISIN.csv" % i), key="ISIN"),
+    True
+)
 
-print(template.render(added=added, changed=changed, removed=removed))
+print(template.render(added=diff['added'], changed=diff['changed'], removed=diff['removed']))

fetch.sh

@@ -17,7 +17,6 @@ function fetch_page() {
   echo "[+] $1/$2"
   curl "https://nsdl.co.in/master_search_res.php" \
     --no-progress-meter \
-    --write-out '%{stderr}DL  :%{size_download}\nHTTP:%{response_code}\n' \
     --user-agent "Mozilla/Gecko/Firefox/58.0" \
     --retry 10 \
     --connect-timeout 30 \
@@ -53,6 +52,7 @@ CLASS="$1"
 
 total=$(fetch_total_pages "$CLASS")
 echo "::group::$CLASS (Total=$total)"
+rm "$CLASS.csv"
 fetch_class "$CLASS" $total
 echo "::endgroup::"
 
@@ -63,17 +63,3 @@ sort -o "$CLASS.csv" "$CLASS.csv"
 # Remove lines that don't start with the correct prefix
 # This is to avoid ISINs like INF955L01IN9 showing up under IN9
 sed -i "/^$CLASS/!d" "$CLASS.csv"
-
-# Now add a header
-cat header.csv $CLASS.csv > tmp.csv
-mv tmp.csv $CLASS.csv
-
-# Update CITATION
-if [[ $(git diff --stat *.csv) != '' ]]; then
-  sed -i "s/^version.*/version: $1/" CITATION.cff
-  sed -i "s/^date-released.*/date-released: $(date --rfc-3339=date)/" CITATION.cff
-
-  jq ".version = \"$1\" | .created = \"$(date --rfc-3339=seconds)\"" datapackage.json > d2.json
-  mv d2.json datapackage.json
-  git add CITATION.cff datapackage.json
-fi