I need to write some code that checks thousands of websites, to determine if they are in English or not.Below is the source code. Any improvements would be appreciated.import nltkimport urllib2import reimport unicodedataENGLISH_STOPWORDS = set(nltk.corpus.stopwords.words('english'))NON_ENGLISH_STOPWORDS = set(nltk.corpus.stopwords.words()) - ENGLISH_STOPWORDSSTOPWORDS_DICT = {lang: set(nltk.corpus.stopwords.words(lang)) for lang in nltk.corpus.stopwords.fileids()}def get_language(text):    words = set(nltk.wordpunct_tokenize(text.lower()))    return max(((lang, len(words & stopwords)) for lang, stopwords in STOPWORDS_DICT.items()), key=lambda x: x[1])[0]def checkEnglish(text):    if text is None:        return 0    else:        text = unicode(text, errors='replace')        text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore')        text = text.lower()    words = set(nltk.wordpunct_tokenize(text))    if len(words & ENGLISH_STOPWORDS) > len(words & NON_ENGLISH_STOPWORDS):        return 1    else:        return 0def getPage(url):    if not url.startswith(http://):        url = http:// + url    print Checking the site , url    req = urllib2.Request(url)    try:        response = urllib2.urlopen(req)        rstPage = response.read()    except urllib2.HTTPError, e:        rstPage = None    except urllib2.URLError, e:        rstPage = None    except Exception, e:        rstPage = None    return rstPagedef getPtag(webPage):    if webPage is None:        return None    else:        rst = re.search(r'<p\W*(.+)\W*</p>', webPage)        if rst is not None:            return rst.group(1)        else:            return rstdef getDescription(webPage):    if webPage is None:        return None    else:        des = re.search(r'<meta\s+.+\[Dd]escription\\s+content=\(.+)\\s*/*>', webPage)        if des is not None:            return des.group(1)        else:            return desdef checking(url):    pageText = getPage(url)    if pageText is not None:        if checkEnglish(getDescription(pageText)) == 1:            return '1'        elif checkEnglish(getPtag(pageText)) == 1:            return '1'        elif checkEnglish(pageText) == 1:            return '1'        else:            return '0'    else:        return 'NULL'if __name__ == __main__:    f = open('sample_domain_list.txt').readlines()    s = open('newestResult.txt', w)    for line in f[:20]:        url = line.split(',')[1][1:-1]        check = checking(url)        s.write(url + ',' + check)        s.write('\n')        print check#    f.close()    s.close()
NLTK language detection code in Python
python;parsing;natural language processing