From 8f9a7c9c1d183312a3b4089d37e0763fc8f90ef8 Mon Sep 17 00:00:00 2001
From: Nemo <commits@captnemo.in>
Date: Thu, 19 May 2022 11:46:10 +0530
Subject: [PATCH] Single file release process

---
 .github/workflows/update.yml |  6 +++---
 .gitignore                   |  1 +
 Makefile                     | 38 +++++++++++++++++-------------------
 diff.py                      | 17 ++++++----------
 fetch.sh                     | 16 +--------------
 5 files changed, 29 insertions(+), 49 deletions(-)

diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml
index 6b3777a..5c5abb7 100644
--- a/.github/workflows/update.yml
+++ b/.github/workflows/update.yml
@@ -14,15 +14,15 @@ jobs:
       with:
         ref: ${{ github.head_ref }}
     - name: Update data
-      run: make all
+      run: make update
       id: update_data
     # Only tag if we're running on the scheduled job
     - uses: stefanzweifel/git-auto-commit-action@v4
       with:
         commit_message: Update ISIN Data
         commit_author: 'github-actions[bot] <github-actions[bot]@users.noreply.github.com>'
-        file_pattern: "IN*.csv"
-        status_options: '--untracked-files=normal'
+        file_pattern: "ISIN.csv"
+        status_options: '--untracked-files=no'
         tagging_message: "v${{ steps.update_data.outputs.version }}"
     - name: Create Release
       run: |
diff --git a/.gitignore b/.gitignore
index 5947563..17dfcb8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 pup
 release.md
 pup.zip
+IN*.csv
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 58999c7..bd9fd53 100644
--- a/Makefile
+++ b/Makefile
@@ -1,32 +1,30 @@
 SHELL=/bin/bash
 version=`date +%Y.%-m.%-d`
 
-all: INE INF IN9 IN0 IN1 IN2 IN3 IN4 update
+# Build Process
+
+ISIN: INE INF IN9 IN0 IN1 IN2 IN3 IN4
+	cat header.csv IN*.csv > ISIN.csv
+	rm IN*.csv
 
 INE INF IN9 IN0 IN1 IN2 IN3 IN4:
 	./fetch.sh $@
-
-old:
-	git show HEAD^:INE.csv > /tmp/INE.csv
-	git show HEAD^:INF.csv > /tmp/INF.csv
-	git show HEAD^:IN9.csv > /tmp/IN9.csv
-	git show HEAD^:IN0.csv > /tmp/IN0.csv
-	git show HEAD^:IN1.csv > /tmp/IN1.csv
-	git show HEAD^:IN2.csv > /tmp/IN2.csv
-	git show HEAD^:IN3.csv > /tmp/IN3.csv
-	git show HEAD^:IN4.csv > /tmp/IN4.csv
-
-release.md: old
-	python3 diff.py > release.md
-
-release: release.md
-	gh release create "$(version)" --notes-file release.md IN*.csv
-
-update:
+update: ISIN
 	echo "::set-output name=version::$(version)"
 	sed -i "s/^version.*/version: $(version)/" CITATION.cff
 	sed -i "s/^date-released.*/date-released: `date --rfc-3339=date`/" CITATION.cff
 
 	jq ".version = \"$(version)\" | .created = \"`date --rfc-3339=seconds`\"" datapackage.json > d2.json
 	mv d2.json datapackage.json
-	git add CITATION.cff datapackage.json
\ No newline at end of file
+	git add CITATION.cff datapackage.json
+
+# Release Process
+
+old:
+	git show HEAD^:ISIN.csv > /tmp/ISIN.csv
+
+release.md: old
+	python3 diff.py > release.md
+
+release: release.md
+	gh release create "$(version)" --notes-file release.md ISIN.csv
diff --git a/diff.py b/diff.py
index 0a8bea8..935faa6 100644
--- a/diff.py
+++ b/diff.py
@@ -11,15 +11,10 @@ def chunks(lst, n):
     for i in range(0, len(lst), n):
         yield lst[i:i + n]
 
-for i in ['E', 'F', '9', '0', '1', '2', '3', '4']:
-    diff = compare(
-        load_csv(open("/tmp/IN%s.csv" % i), key="ISIN"),
-        load_csv(open("IN%s.csv" % i), key="ISIN"),
-        True
-    )
-    # print(diff)
-    added += diff['added']
-    changed += diff['changed']
-    removed += diff['removed']
+diff = compare(
+    load_csv(open("/tmp/ISIN.csv" % i), key="ISIN"),
+    load_csv(open("ISIN.csv" % i), key="ISIN"),
+    True
+)
 
-print(template.render(added=added, changed=changed, removed=removed))
\ No newline at end of file
+print(template.render(added=diff['added'], changed=diff['changed'], removed=diff['removed']))
\ No newline at end of file
diff --git a/fetch.sh b/fetch.sh
index ef213d7..9d771fc 100755
--- a/fetch.sh
+++ b/fetch.sh
@@ -17,7 +17,6 @@ function fetch_page() {
   echo "[+] $1/$2"
   curl "https://nsdl.co.in/master_search_res.php" \
     --no-progress-meter \
-    --write-out '%{stderr}DL  :%{size_download}\nHTTP:%{response_code}\n' \
     --user-agent "Mozilla/Gecko/Firefox/58.0" \
     --retry 10 \
     --connect-timeout 30 \
@@ -53,6 +52,7 @@ CLASS="$1"
 
 total=$(fetch_total_pages "$CLASS")
 echo "::group::$CLASS (Total=$total)"
+rm "$CLASS.csv"
 fetch_class "$CLASS" $total
 echo "::endgroup::"
 
@@ -63,17 +63,3 @@ sort -o "$CLASS.csv" "$CLASS.csv"
 # Remove lines that don't start with the correct prefix
 # This is to avoid ISINs like INF955L01IN9 showing up under IN9
 sed -i "/^$CLASS/!d" "$CLASS.csv"
-
-# Now add a header
-cat header.csv $CLASS.csv > tmp.csv
-mv tmp.csv $CLASS.csv
-
-# Update CITATION
-if [[ $(git diff --stat *.csv) != '' ]]; then
-  sed -i "s/^version.*/version: $1/" CITATION.cff
-  sed -i "s/^date-released.*/date-released: $(date --rfc-3339=date)/" CITATION.cff
-
-  jq ".version = \"$1\" | .created = \"$(date --rfc-3339=seconds)\"" datapackage.json > d2.json
-  mv d2.json datapackage.json
-  git add CITATION.cff datapackage.json
-fi