From 5dd903adde23c608962a2cae2833cb5b0af7e28e Mon Sep 17 00:00:00 2001 From: Nemo Date: Sat, 14 May 2022 18:30:02 +0530 Subject: [PATCH] [ci] Improve the final release artifact --- .github/workflows/update.yml | 8 +--- .gitignore | 3 +- Makefile | 11 +++++ datapackage.json | 24 +++++----- diff.py | 25 +++++++++++ fetch.sh | 2 +- header.csv | 2 +- release.mako | 85 ++++++++++++++++++++++++++++++++++++ requirements.txt | 6 +++ 9 files changed, 145 insertions(+), 21 deletions(-) create mode 100644 diff.py create mode 100644 release.mako create mode 100644 requirements.txt diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 4d691f7..1e05d7e 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -25,11 +25,7 @@ jobs: tagging_message: "v${{ steps.update_data.outputs.version }}" - name: Create Release run: | - git diff --ignore-all-space -U0 HEAD^ *.csv|egrep -oe '^([+-]IN\w{10})' > /tmp/release.txt - echo "$(cat /tmp/release.txt | wc -l) Changes" > /tmp/release.md - echo '```diff' >> /tmp/release.md - cat /tmp/release.txt >> /tmp/release.md - echo '```' >> /tmp/release.md - gh release create "v${{ steps.update_data.outputs.version }}" --notes-file /tmp/release.md *.csv + pip install -r requirements.txt + make release "version=v${{ steps.update_data.outputs.version }}" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 17876e8..5947563 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ pup -pup.zip \ No newline at end of file +release.md +pup.zip diff --git a/Makefile b/Makefile index 9a67afa..8692d26 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,17 @@ all: INE INF IN9 update INE INF IN9: ./fetch.sh $@ +old: + git show HEAD^:INE.csv > /tmp/INE.csv + git show HEAD^:INF.csv > /tmp/INF.csv + git show HEAD^:IN9.csv > /tmp/IN9.csv + +release.md: old + python3 diff.py > release.md + +release: release.md + gh release create "$(version)" --notes-file release.md IN*.csv + update: echo "::set-output name=version::$(version)" sed -i "s/^version.*/version: $(version)/" CITATION.cff diff --git a/datapackage.json b/datapackage.json index 12c8729..a3c9166 100644 --- a/datapackage.json +++ b/datapackage.json @@ -40,14 +40,14 @@ }, { "description": "Description for the security", - "example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH", + "example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH", "name": "Description", - "title": "ISIN Description", + "title": "Security Description", "type": "string" }, { "description": "Name of the issuer that issued this security", - "example": "ITI MUTUAL FUND", + "example": "UTI MUTUAL FUND", "name": "Issuer", "title": "Name of the Issuer", "type": "string" @@ -56,7 +56,7 @@ "description": "Description of the type of security", "example": "MUTUAL FUND UNIT (TRASE)", "name": "Type", - "title": "Security Type", + "title": "Issuer Type", "type": "string" }, { @@ -88,14 +88,14 @@ }, { "description": "Description for the security", - "example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH", + "example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH", "name": "Description", - "title": "ISIN Description", + "title": "Security Description", "type": "string" }, { "description": "Name of the issuer that issued this security", - "example": "ITI MUTUAL FUND", + "example": "UTI MUTUAL FUND", "name": "Issuer", "title": "Name of the Issuer", "type": "string" @@ -104,7 +104,7 @@ "description": "Description of the type of security", "example": "MUTUAL FUND UNIT (TRASE)", "name": "Type", - "title": "Security Type", + "title": "Issuer Type", "type": "string" }, { @@ -136,14 +136,14 @@ }, { "description": "Description for the security", - "example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH", + "example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH", "name": "Description", - "title": "ISIN Description", + "title": "Security Description", "type": "string" }, { "description": "Name of the issuer that issued this security", - "example": "ITI MUTUAL FUND", + "example": "UTI MUTUAL FUND", "name": "Issuer", "title": "Name of the Issuer", "type": "string" @@ -152,7 +152,7 @@ "description": "Description of the type of security", "example": "MUTUAL FUND UNIT (TRASE)", "name": "Type", - "title": "Security Type", + "title": "Issuer Type", "type": "string" }, { diff --git a/diff.py b/diff.py new file mode 100644 index 0000000..0b1cfc4 --- /dev/null +++ b/diff.py @@ -0,0 +1,25 @@ +from csv_diff import load_csv, compare +import json +from mako.template import Template + +template = Template(filename='release.mako') + +added,removed,changed = [],[],[] + +def chunks(lst, n): + """Yield successive n-sized chunks from lst.""" + for i in range(0, len(lst), n): + yield lst[i:i + n] + +for i in ['E', 'F', '9']: + diff = compare( + load_csv(open("/tmp/IN%s.csv" % i), key="ISIN"), + load_csv(open("IN%s.csv" % i), key="ISIN"), + True + ) + # print(diff) + added += diff['added'] + changed += diff['changed'] + removed += diff['removed'] + +print(template.render(added=added, changed=changed, removed=removed)) \ No newline at end of file diff --git a/fetch.sh b/fetch.sh index e2c0750..076dd5f 100755 --- a/fetch.sh +++ b/fetch.sh @@ -53,7 +53,7 @@ CLASS="$1" total=$(fetch_total_pages "$CLASS") echo "::group::$CLASS (Total=$total)" -rm "$CLASS.csv" +cp header.csv "$CLASS.csv" fetch_class "$CLASS" $total echo "::endgroup::" diff --git a/header.csv b/header.csv index dee9a4c..55ff78a 100644 --- a/header.csv +++ b/header.csv @@ -1 +1 @@ -ISIN,Description,Issuer,"Security Type",Status +ISIN,Description,Issuer,Type,Status diff --git a/release.mako b/release.mako new file mode 100644 index 0000000..75e2f31 --- /dev/null +++ b/release.mako @@ -0,0 +1,85 @@ +## Generates a single diff for a single row, ignoring the Status field +<%def name="singlediff(row)"> +@@ ${row['key']} +%for k in row['changes']: +%if k != 'Status': +-${k}:${row['changes'][k][0]} ++${k}:${row['changes'][k][1]} +%endif +%endfor + + +## Get description from a changed row +## Either from the changed or unchanged portion +<%def name="description(row)"> +% if 'Description' in row['unchanged']: +${row['unchanged']['Description']} +%else: +${row['changes']['Description'][1]} +%endif + + +

Additions

+ +The following new ISINs were added: + +ISIN|Description|Issuer|Type|Status +----|-----------|------|---------------|------ +% for row in added: +`${row['ISIN']}`|${row['Description']}|${row['Issuer']}|${row['Type']}|${row['Status'].title()} +% endfor + +

Changes

+ +The following ISINs changed their status: + + + + + + + + + + + +% for row in changed: +% if 'Status' in row['changes']: + + + + + + +% endif +% endfor + +
ISINDescriptionOld StatusNew Status
${row['key']}${description(row)}${row['changes']['Status'][0].title()}${row['changes']['Status'][1].title()}
+ +## This will usually contain the description + +The following ISINs changed other fields: + +```diff +% for row in changed: +% if 'Status' not in row['changes']: +${singlediff(row)} +% endif +% endfor +``` + +## Removals are currently happening accidentally because NSDL website returns a 5xx + +%if len(removed) > 0: + +

Removals

+ +The following ISINs were completely removed (likely in error): + +ISIN|Description|Issuer|Type|Status +----|-----------|------|---------------|------ +% for row in removed: +`${row['ISIN']}`|${row['Description']}|${row['Issuer']}|${row['Type']}|${row['Status'].title()} +% endfor + +%endif \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9a1802b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +click==8.1.3 +csv-diff==1.1 +dictdiffer==0.9.0 +Mako==1.2.0 +MarkupSafe==2.1.1 +six==1.16.0