[ci] Improve the final release artifact
This commit is contained in:
parent
9114e918a0
commit
5dd903adde
|
|
@ -25,11 +25,7 @@ jobs:
|
|||
tagging_message: "v${{ steps.update_data.outputs.version }}"
|
||||
- name: Create Release
|
||||
run: |
|
||||
git diff --ignore-all-space -U0 HEAD^ *.csv|egrep -oe '^([+-]IN\w{10})' > /tmp/release.txt
|
||||
echo "$(cat /tmp/release.txt | wc -l) Changes" > /tmp/release.md
|
||||
echo '```diff' >> /tmp/release.md
|
||||
cat /tmp/release.txt >> /tmp/release.md
|
||||
echo '```' >> /tmp/release.md
|
||||
gh release create "v${{ steps.update_data.outputs.version }}" --notes-file /tmp/release.md *.csv
|
||||
pip install -r requirements.txt
|
||||
make release "version=v${{ steps.update_data.outputs.version }}"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
pup
|
||||
pup.zip
|
||||
release.md
|
||||
pup.zip
|
||||
|
|
|
|||
11
Makefile
11
Makefile
|
|
@ -5,6 +5,17 @@ all: INE INF IN9 update
|
|||
INE INF IN9:
|
||||
./fetch.sh $@
|
||||
|
||||
old:
|
||||
git show HEAD^:INE.csv > /tmp/INE.csv
|
||||
git show HEAD^:INF.csv > /tmp/INF.csv
|
||||
git show HEAD^:IN9.csv > /tmp/IN9.csv
|
||||
|
||||
release.md: old
|
||||
python3 diff.py > release.md
|
||||
|
||||
release: release.md
|
||||
gh release create "$(version)" --notes-file release.md IN*.csv
|
||||
|
||||
update:
|
||||
echo "::set-output name=version::$(version)"
|
||||
sed -i "s/^version.*/version: $(version)/" CITATION.cff
|
||||
|
|
|
|||
|
|
@ -40,14 +40,14 @@
|
|||
},
|
||||
{
|
||||
"description": "Description for the security",
|
||||
"example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||
"example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||
"name": "Description",
|
||||
"title": "ISIN Description",
|
||||
"title": "Security Description",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Name of the issuer that issued this security",
|
||||
"example": "ITI MUTUAL FUND",
|
||||
"example": "UTI MUTUAL FUND",
|
||||
"name": "Issuer",
|
||||
"title": "Name of the Issuer",
|
||||
"type": "string"
|
||||
|
|
@ -56,7 +56,7 @@
|
|||
"description": "Description of the type of security",
|
||||
"example": "MUTUAL FUND UNIT (TRASE)",
|
||||
"name": "Type",
|
||||
"title": "Security Type",
|
||||
"title": "Issuer Type",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
|
|
@ -88,14 +88,14 @@
|
|||
},
|
||||
{
|
||||
"description": "Description for the security",
|
||||
"example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||
"example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||
"name": "Description",
|
||||
"title": "ISIN Description",
|
||||
"title": "Security Description",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Name of the issuer that issued this security",
|
||||
"example": "ITI MUTUAL FUND",
|
||||
"example": "UTI MUTUAL FUND",
|
||||
"name": "Issuer",
|
||||
"title": "Name of the Issuer",
|
||||
"type": "string"
|
||||
|
|
@ -104,7 +104,7 @@
|
|||
"description": "Description of the type of security",
|
||||
"example": "MUTUAL FUND UNIT (TRASE)",
|
||||
"name": "Type",
|
||||
"title": "Security Type",
|
||||
"title": "Issuer Type",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
|
|
@ -136,14 +136,14 @@
|
|||
},
|
||||
{
|
||||
"description": "Description for the security",
|
||||
"example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||
"example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||
"name": "Description",
|
||||
"title": "ISIN Description",
|
||||
"title": "Security Description",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"description": "Name of the issuer that issued this security",
|
||||
"example": "ITI MUTUAL FUND",
|
||||
"example": "UTI MUTUAL FUND",
|
||||
"name": "Issuer",
|
||||
"title": "Name of the Issuer",
|
||||
"type": "string"
|
||||
|
|
@ -152,7 +152,7 @@
|
|||
"description": "Description of the type of security",
|
||||
"example": "MUTUAL FUND UNIT (TRASE)",
|
||||
"name": "Type",
|
||||
"title": "Security Type",
|
||||
"title": "Issuer Type",
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
from csv_diff import load_csv, compare
|
||||
import json
|
||||
from mako.template import Template
|
||||
|
||||
template = Template(filename='release.mako')
|
||||
|
||||
added,removed,changed = [],[],[]
|
||||
|
||||
def chunks(lst, n):
|
||||
"""Yield successive n-sized chunks from lst."""
|
||||
for i in range(0, len(lst), n):
|
||||
yield lst[i:i + n]
|
||||
|
||||
for i in ['E', 'F', '9']:
|
||||
diff = compare(
|
||||
load_csv(open("/tmp/IN%s.csv" % i), key="ISIN"),
|
||||
load_csv(open("IN%s.csv" % i), key="ISIN"),
|
||||
True
|
||||
)
|
||||
# print(diff)
|
||||
added += diff['added']
|
||||
changed += diff['changed']
|
||||
removed += diff['removed']
|
||||
|
||||
print(template.render(added=added, changed=changed, removed=removed))
|
||||
2
fetch.sh
2
fetch.sh
|
|
@ -53,7 +53,7 @@ CLASS="$1"
|
|||
|
||||
total=$(fetch_total_pages "$CLASS")
|
||||
echo "::group::$CLASS (Total=$total)"
|
||||
rm "$CLASS.csv"
|
||||
cp header.csv "$CLASS.csv"
|
||||
fetch_class "$CLASS" $total
|
||||
echo "::endgroup::"
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
ISIN,Description,Issuer,"Security Type",Status
|
||||
ISIN,Description,Issuer,Type,Status
|
||||
|
|
|
|||
|
|
|
@ -0,0 +1,85 @@
|
|||
## Generates a single diff for a single row, ignoring the Status field
|
||||
<%def name="singlediff(row)">
|
||||
@@ ${row['key']}
|
||||
%for k in row['changes']:
|
||||
%if k != 'Status':
|
||||
-${k}:${row['changes'][k][0]}
|
||||
+${k}:${row['changes'][k][1]}
|
||||
%endif
|
||||
%endfor
|
||||
</%def>
|
||||
|
||||
## Get description from a changed row
|
||||
## Either from the changed or unchanged portion
|
||||
<%def name="description(row)">
|
||||
% if 'Description' in row['unchanged']:
|
||||
${row['unchanged']['Description']}
|
||||
%else:
|
||||
${row['changes']['Description'][1]}
|
||||
%endif
|
||||
</%def>
|
||||
|
||||
<h2>Additions</h2>
|
||||
|
||||
The following new ISINs were added:
|
||||
|
||||
ISIN|Description|Issuer|Type|Status
|
||||
----|-----------|------|---------------|------
|
||||
% for row in added:
|
||||
`${row['ISIN']}`|${row['Description']}|${row['Issuer']}|${row['Type']}|${row['Status'].title()}
|
||||
% endfor
|
||||
|
||||
<h2>Changes</h2>
|
||||
|
||||
The following ISINs changed their status:
|
||||
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>ISIN</th>
|
||||
<th>Description</th>
|
||||
<th>Old Status</th>
|
||||
<th>New Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
% for row in changed:
|
||||
% if 'Status' in row['changes']:
|
||||
<tr>
|
||||
<td><code>${row['key']}</code></td>
|
||||
<td>${description(row)}</td>
|
||||
<td>${row['changes']['Status'][0].title()}</td>
|
||||
<td>${row['changes']['Status'][1].title()}</td>
|
||||
</tr>
|
||||
% endif
|
||||
% endfor
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
## This will usually contain the description
|
||||
|
||||
The following ISINs changed other fields:
|
||||
|
||||
```diff
|
||||
% for row in changed:
|
||||
% if 'Status' not in row['changes']:
|
||||
${singlediff(row)}
|
||||
% endif
|
||||
% endfor
|
||||
```
|
||||
|
||||
## Removals are currently happening accidentally because NSDL website returns a 5xx
|
||||
|
||||
%if len(removed) > 0:
|
||||
|
||||
<h2>Removals</h2>
|
||||
|
||||
The following ISINs were completely removed (likely in error):
|
||||
|
||||
ISIN|Description|Issuer|Type|Status
|
||||
----|-----------|------|---------------|------
|
||||
% for row in removed:
|
||||
`${row['ISIN']}`|${row['Description']}|${row['Issuer']}|${row['Type']}|${row['Status'].title()}
|
||||
% endfor
|
||||
|
||||
%endif
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
click==8.1.3
|
||||
csv-diff==1.1
|
||||
dictdiffer==0.9.0
|
||||
Mako==1.2.0
|
||||
MarkupSafe==2.1.1
|
||||
six==1.16.0
|
||||
Loading…
Reference in New Issue