Initial commit

.gitignore

@@ -0,0 +1,2 @@
+cache/
+*.txt

README.md

@@ -0,0 +1,3 @@
+# pulse-in
+
+How many Indian government websites are using HTTPS? An attempt at tracking them using [18F/pulse](https://github.com/18F/pulse#setup).

dl.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Download from the goidirectory website
+for i in $(seq 1 14388); do
+	echo $i
+  curl -s "http://goidirectory.nic.in/sitecounter.php?id=$i" |grep  '<meta http-equiv="refresh"' >> urls.raw.txt
+done
+
+# Get crts issued in gov.in
+# TODO: This isn't as good as the censys.io export
+# that pulse uses, switch to that
+echo "select name_value FROM certificate_identity WHERE name_value LIKE '%.gov.in';" | psql -h crt.sh -p 5432 -U guest certwatch >  crt.sh.domains.txt
+
+cat urls.raw.txt crt.sh.domains.txt |sort -u > combined.txt
+
+php parse.php | sort -u > domains.csv

parse.php

@@ -0,0 +1,8 @@
+<?php
+
+foreach(file('combined.txt') as $row)
+{
+	$row = trim($row);
+	$url = "http://$row/";
+	echo parse_url($url)['host'] . PHP_EOL;
+}