[ci] Improve the final release artifact
This commit is contained in:
parent
9114e918a0
commit
5dd903adde
|
@ -25,11 +25,7 @@ jobs:
|
||||||
tagging_message: "v${{ steps.update_data.outputs.version }}"
|
tagging_message: "v${{ steps.update_data.outputs.version }}"
|
||||||
- name: Create Release
|
- name: Create Release
|
||||||
run: |
|
run: |
|
||||||
git diff --ignore-all-space -U0 HEAD^ *.csv|egrep -oe '^([+-]IN\w{10})' > /tmp/release.txt
|
pip install -r requirements.txt
|
||||||
echo "$(cat /tmp/release.txt | wc -l) Changes" > /tmp/release.md
|
make release "version=v${{ steps.update_data.outputs.version }}"
|
||||||
echo '```diff' >> /tmp/release.md
|
|
||||||
cat /tmp/release.txt >> /tmp/release.md
|
|
||||||
echo '```' >> /tmp/release.md
|
|
||||||
gh release create "v${{ steps.update_data.outputs.version }}" --notes-file /tmp/release.md *.csv
|
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
|
|
@ -1,2 +1,3 @@
|
||||||
pup
|
pup
|
||||||
pup.zip
|
release.md
|
||||||
|
pup.zip
|
||||||
|
|
11
Makefile
11
Makefile
|
@ -5,6 +5,17 @@ all: INE INF IN9 update
|
||||||
INE INF IN9:
|
INE INF IN9:
|
||||||
./fetch.sh $@
|
./fetch.sh $@
|
||||||
|
|
||||||
|
old:
|
||||||
|
git show HEAD^:INE.csv > /tmp/INE.csv
|
||||||
|
git show HEAD^:INF.csv > /tmp/INF.csv
|
||||||
|
git show HEAD^:IN9.csv > /tmp/IN9.csv
|
||||||
|
|
||||||
|
release.md: old
|
||||||
|
python3 diff.py > release.md
|
||||||
|
|
||||||
|
release: release.md
|
||||||
|
gh release create "$(version)" --notes-file release.md IN*.csv
|
||||||
|
|
||||||
update:
|
update:
|
||||||
echo "::set-output name=version::$(version)"
|
echo "::set-output name=version::$(version)"
|
||||||
sed -i "s/^version.*/version: $(version)/" CITATION.cff
|
sed -i "s/^version.*/version: $(version)/" CITATION.cff
|
||||||
|
|
|
@ -40,14 +40,14 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Description for the security",
|
"description": "Description for the security",
|
||||||
"example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH",
|
"example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||||
"name": "Description",
|
"name": "Description",
|
||||||
"title": "ISIN Description",
|
"title": "Security Description",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Name of the issuer that issued this security",
|
"description": "Name of the issuer that issued this security",
|
||||||
"example": "ITI MUTUAL FUND",
|
"example": "UTI MUTUAL FUND",
|
||||||
"name": "Issuer",
|
"name": "Issuer",
|
||||||
"title": "Name of the Issuer",
|
"title": "Name of the Issuer",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
@ -56,7 +56,7 @@
|
||||||
"description": "Description of the type of security",
|
"description": "Description of the type of security",
|
||||||
"example": "MUTUAL FUND UNIT (TRASE)",
|
"example": "MUTUAL FUND UNIT (TRASE)",
|
||||||
"name": "Type",
|
"name": "Type",
|
||||||
"title": "Security Type",
|
"title": "Issuer Type",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -88,14 +88,14 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Description for the security",
|
"description": "Description for the security",
|
||||||
"example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH",
|
"example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||||
"name": "Description",
|
"name": "Description",
|
||||||
"title": "ISIN Description",
|
"title": "Security Description",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Name of the issuer that issued this security",
|
"description": "Name of the issuer that issued this security",
|
||||||
"example": "ITI MUTUAL FUND",
|
"example": "UTI MUTUAL FUND",
|
||||||
"name": "Issuer",
|
"name": "Issuer",
|
||||||
"title": "Name of the Issuer",
|
"title": "Name of the Issuer",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
@ -104,7 +104,7 @@
|
||||||
"description": "Description of the type of security",
|
"description": "Description of the type of security",
|
||||||
"example": "MUTUAL FUND UNIT (TRASE)",
|
"example": "MUTUAL FUND UNIT (TRASE)",
|
||||||
"name": "Type",
|
"name": "Type",
|
||||||
"title": "Security Type",
|
"title": "Issuer Type",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
@ -136,14 +136,14 @@
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Description for the security",
|
"description": "Description for the security",
|
||||||
"example": "ITI MUTUAL FUND LIQUID FUND REG GROWTH",
|
"example": "UTI MUTUAL FUND LIQUID FUND REG GROWTH",
|
||||||
"name": "Description",
|
"name": "Description",
|
||||||
"title": "ISIN Description",
|
"title": "Security Description",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"description": "Name of the issuer that issued this security",
|
"description": "Name of the issuer that issued this security",
|
||||||
"example": "ITI MUTUAL FUND",
|
"example": "UTI MUTUAL FUND",
|
||||||
"name": "Issuer",
|
"name": "Issuer",
|
||||||
"title": "Name of the Issuer",
|
"title": "Name of the Issuer",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
|
@ -152,7 +152,7 @@
|
||||||
"description": "Description of the type of security",
|
"description": "Description of the type of security",
|
||||||
"example": "MUTUAL FUND UNIT (TRASE)",
|
"example": "MUTUAL FUND UNIT (TRASE)",
|
||||||
"name": "Type",
|
"name": "Type",
|
||||||
"title": "Security Type",
|
"title": "Issuer Type",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
@ -0,0 +1,25 @@
|
||||||
|
from csv_diff import load_csv, compare
|
||||||
|
import json
|
||||||
|
from mako.template import Template
|
||||||
|
|
||||||
|
template = Template(filename='release.mako')
|
||||||
|
|
||||||
|
added,removed,changed = [],[],[]
|
||||||
|
|
||||||
|
def chunks(lst, n):
|
||||||
|
"""Yield successive n-sized chunks from lst."""
|
||||||
|
for i in range(0, len(lst), n):
|
||||||
|
yield lst[i:i + n]
|
||||||
|
|
||||||
|
for i in ['E', 'F', '9']:
|
||||||
|
diff = compare(
|
||||||
|
load_csv(open("/tmp/IN%s.csv" % i), key="ISIN"),
|
||||||
|
load_csv(open("IN%s.csv" % i), key="ISIN"),
|
||||||
|
True
|
||||||
|
)
|
||||||
|
# print(diff)
|
||||||
|
added += diff['added']
|
||||||
|
changed += diff['changed']
|
||||||
|
removed += diff['removed']
|
||||||
|
|
||||||
|
print(template.render(added=added, changed=changed, removed=removed))
|
2
fetch.sh
2
fetch.sh
|
@ -53,7 +53,7 @@ CLASS="$1"
|
||||||
|
|
||||||
total=$(fetch_total_pages "$CLASS")
|
total=$(fetch_total_pages "$CLASS")
|
||||||
echo "::group::$CLASS (Total=$total)"
|
echo "::group::$CLASS (Total=$total)"
|
||||||
rm "$CLASS.csv"
|
cp header.csv "$CLASS.csv"
|
||||||
fetch_class "$CLASS" $total
|
fetch_class "$CLASS" $total
|
||||||
echo "::endgroup::"
|
echo "::endgroup::"
|
||||||
|
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
ISIN,Description,Issuer,"Security Type",Status
|
ISIN,Description,Issuer,Type,Status
|
||||||
|
|
|
|
@ -0,0 +1,85 @@
|
||||||
|
## Generates a single diff for a single row, ignoring the Status field
|
||||||
|
<%def name="singlediff(row)">
|
||||||
|
@@ ${row['key']}
|
||||||
|
%for k in row['changes']:
|
||||||
|
%if k != 'Status':
|
||||||
|
-${k}:${row['changes'][k][0]}
|
||||||
|
+${k}:${row['changes'][k][1]}
|
||||||
|
%endif
|
||||||
|
%endfor
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
## Get description from a changed row
|
||||||
|
## Either from the changed or unchanged portion
|
||||||
|
<%def name="description(row)">
|
||||||
|
% if 'Description' in row['unchanged']:
|
||||||
|
${row['unchanged']['Description']}
|
||||||
|
%else:
|
||||||
|
${row['changes']['Description'][1]}
|
||||||
|
%endif
|
||||||
|
</%def>
|
||||||
|
|
||||||
|
<h2>Additions</h2>
|
||||||
|
|
||||||
|
The following new ISINs were added:
|
||||||
|
|
||||||
|
ISIN|Description|Issuer|Type|Status
|
||||||
|
----|-----------|------|---------------|------
|
||||||
|
% for row in added:
|
||||||
|
`${row['ISIN']}`|${row['Description']}|${row['Issuer']}|${row['Type']}|${row['Status'].title()}
|
||||||
|
% endfor
|
||||||
|
|
||||||
|
<h2>Changes</h2>
|
||||||
|
|
||||||
|
The following ISINs changed their status:
|
||||||
|
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ISIN</th>
|
||||||
|
<th>Description</th>
|
||||||
|
<th>Old Status</th>
|
||||||
|
<th>New Status</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
% for row in changed:
|
||||||
|
% if 'Status' in row['changes']:
|
||||||
|
<tr>
|
||||||
|
<td><code>${row['key']}</code></td>
|
||||||
|
<td>${description(row)}</td>
|
||||||
|
<td>${row['changes']['Status'][0].title()}</td>
|
||||||
|
<td>${row['changes']['Status'][1].title()}</td>
|
||||||
|
</tr>
|
||||||
|
% endif
|
||||||
|
% endfor
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
## This will usually contain the description
|
||||||
|
|
||||||
|
The following ISINs changed other fields:
|
||||||
|
|
||||||
|
```diff
|
||||||
|
% for row in changed:
|
||||||
|
% if 'Status' not in row['changes']:
|
||||||
|
${singlediff(row)}
|
||||||
|
% endif
|
||||||
|
% endfor
|
||||||
|
```
|
||||||
|
|
||||||
|
## Removals are currently happening accidentally because NSDL website returns a 5xx
|
||||||
|
|
||||||
|
%if len(removed) > 0:
|
||||||
|
|
||||||
|
<h2>Removals</h2>
|
||||||
|
|
||||||
|
The following ISINs were completely removed (likely in error):
|
||||||
|
|
||||||
|
ISIN|Description|Issuer|Type|Status
|
||||||
|
----|-----------|------|---------------|------
|
||||||
|
% for row in removed:
|
||||||
|
`${row['ISIN']}`|${row['Description']}|${row['Issuer']}|${row['Type']}|${row['Status'].title()}
|
||||||
|
% endfor
|
||||||
|
|
||||||
|
%endif
|
|
@ -0,0 +1,6 @@
|
||||||
|
click==8.1.3
|
||||||
|
csv-diff==1.1
|
||||||
|
dictdiffer==0.9.0
|
||||||
|
Mako==1.2.0
|
||||||
|
MarkupSafe==2.1.1
|
||||||
|
six==1.16.0
|
Loading…
Reference in New Issue