import urllib2, re

firstLevel = range(1,22)
secondLevel = range(1,21)
count = dict()
allCsv = ""
countCsv = ""

for i in firstLevel:
    for j in secondLevel:
        url = "http://web1.jupas.edu.hk/result_pages/" + str(i) + "-" + str(j) + ".htm"
        try:
            webContent = urllib2.urlopen(url).read()
            results = re.findall(r"<td width=150 align=center id=(\d+)>\d+&nbsp;&nbsp;<a HREF=\"../course_info/(\d+).htm\">\d+</a></td>", webContent)
            for result in results:
                allCsv += '"' + result[0] + '","' + result[1] + '"\n'
                if result[1] in count:
                    count[result[1]] += 1
                else:
                    count[result[1]] = 1
        except:
            # dirty handling of files which are not exist
            pass

# sort by JUPAS code
keys = count.keys()
keys.sort()

i = 0
length = len(count)
while (i < length):
    countCsv += '"' + keys[i] + '","' + str(count[keys[i]]) + '"\n'
    i += 1
    
f = open('all.csv', 'w')
f.write(allCsv)
f.close()
    
f = open('count.csv', 'w')
f.write(countCsv)
f.close()