'''
Created on Nov 29, 2015

@author: ola
modified by rodger, April 25, 2016
modified by rodger, Dec. 7, 2016
'''
import re,urllib2

def scrape(pattern,url):
    source = urllib2.urlopen(url)
    text = source.read()
    #print "read",len(text),"characters"
    for m in re.finditer(pattern,text):
        if m:
            print m.groups()
   
def dukemath():
    # updated website
    pattern = r'mailto:(\w+[.\w]*)@(\w+[.\w+]*)'
    baseurl = "https://math.duke.edu/people/faculty"
    scrape(pattern,baseurl)

def pubpol():
    pattern = r'mailto:(\w+[.\w]*)@(\w+[.\w+]*)'
    baseurl = "http://sanford.duke.edu/people-and-research/directory/a"
    for ch in 'abcdefghijklmnopqrstuvwxyz':
        url = baseurl[:-1]+ch
        print "scraping",ch
        scrape(pattern,url)

        
def biology():
    pattern = r'mailto:(\w+[.\w]*)@(\w+[.\w+]*)'
    baseurl ="https://biology.duke.edu/people/all-faculty/a"
    for ch in 'abcdefghijklmnopqrstuvwxyz':
        url = baseurl[:-1]+ch
        print "scraping",ch
        scrape(pattern,url)
        
if __name__ == '__main__':
    #pubpol()
    dukemath()
    #biology()