Spaces:
Sleeping
Sleeping
| import string,re,sys,os,random,glob | |
| from math import sqrt,log | |
| # adjust minimum sample size here | |
| standard=50 | |
| # Returns the keys of dictionary d sorted by their values | |
| def sort_by_value(d): | |
| items=d.items() | |
| backitems=[ [v[1],v[0]] for v in items] | |
| backitems.sort() | |
| return [ backitems[i][1] for i in range(0,len(backitems))] | |
| # NDW for first z words in a sample | |
| def getndwfirstz(z,lemmalist): | |
| ndwfirstztype={} | |
| for lemma in lemmalist[:z]: | |
| ndwfirstztype[lemma]=1 | |
| return len(ndwfirstztype.keys()) | |
| # NDW expected random z words, 10 trials | |
| def getndwerz(z,lemmalist): | |
| ndwerz=0 | |
| for i in range(10): | |
| ndwerztype={} | |
| erzlemmalist=random.sample(lemmalist,z) | |
| for lemma in erzlemmalist: | |
| ndwerztype[lemma]=1 | |
| ndwerz+=len(ndwerztype.keys()) | |
| return ndwerz/10.0 | |
| # NDW expected random sequences of z words, 10 trials | |
| def getndwesz(z,lemmalist): | |
| ndwesz=0 | |
| for i in range(10): | |
| ndwesztype={} | |
| startword=random.randint(0,len(lemmalist)-z) | |
| eszlemmalist=lemmalist[startword:startword+z] | |
| for lemma in eszlemmalist: | |
| ndwesztype[lemma]=1 | |
| ndwesz+=len(ndwesztype.keys()) | |
| return ndwesz/10.0 | |
| # MSTTR | |
| def getmsttr(z,lemmalist): | |
| samples=0 | |
| msttr=0.0 | |
| while len(lemmalist)>=z: | |
| samples+=1 | |
| msttrtype={} | |
| for lemma in lemmalist[:z]: | |
| msttrtype[lemma]=1 | |
| msttr+=len(msttrtype.keys())/float(z) | |
| lemmalist=lemmalist[z:] | |
| return msttr/samples | |
| def isLetterNumber(character): | |
| if character in string.printable and not character in string.punctuation: | |
| return 1 | |
| return 0 | |
| def isSentence(line): | |
| for character in line: | |
| if isLetterNumber(character): | |
| return 1 | |
| return 0 | |
| def division(x,y): | |
| if y==0: | |
| return 0 | |
| else: | |
| return float(x)/y | |
| # reads information from bnc wordlist | |
| adjdict={} | |
| verbdict={} | |
| noundict={} | |
| worddict={} | |
| wordlistfile=open("bnc_all_filtered.txt","r") | |
| wordlist=wordlistfile.readlines() | |
| wordlistfile.close() | |
| for word in wordlist: | |
| wordinfo=word.strip() | |
| if not wordinfo or "Total words" in wordinfo: | |
| continue | |
| infolist=wordinfo.split() | |
| lemma=infolist[0] | |
| pos=infolist[1] | |
| frequency=int(infolist[2]) | |
| worddict[lemma]=worddict.get(lemma,0)+frequency | |
| if pos=="Adj": | |
| adjdict[lemma]=adjdict.get(lemma,0)+frequency | |
| elif pos=="Verb": | |
| verbdict[lemma]=verbdict.get(lemma,0)+frequency | |
| elif pos=="NoC" or pos=="NoP": | |
| noundict[lemma]=noundict.get(lemma,0)+frequency | |
| wordranks=sort_by_value(worddict) | |
| verbranks=sort_by_value(verbdict) | |
| directoryPath=sys.argv[1] | |
| print "filename, wordtypes, swordtypes, lextypes, slextypes, wordtokens, swordtokens, lextokens, slextokens, ld, ls1, ls2, vs1, vs2, cvs1, ndw, ndwz, ndwerz, ndwesz, ttr, msttr, cttr, rttr, logttr, uber, lv, vv1, svv1, cvv1, vv2, nv, adjv, advv, modv" | |
| for filename in glob.glob( os.path.join(directoryPath, '*') ): | |
| lemfile=open(filename,"r") | |
| lemlines=lemfile.readlines() | |
| lemfile.close() | |
| filename=filename.split("/")[-1] | |
| output=filename | |
| if not lemlines: | |
| output+=",0.0"*31 | |
| print output | |
| continue | |
| # process input file | |
| wordtypes={} | |
| wordtokens=0 | |
| swordtypes={} | |
| swordtokens=0 | |
| lextypes={} | |
| lextokens=0 | |
| slextypes={} | |
| slextokens=0 | |
| verbtypes={} | |
| verbtokens=0 | |
| sverbtypes={} | |
| adjtypes={} | |
| adjtokens=0 | |
| advtypes={} | |
| advtokens=0 | |
| nountypes={} | |
| nountokens=0 | |
| lemmaposlist=[] | |
| lemmalist=[] | |
| for lemline in lemlines: | |
| lemline=lemline.strip() | |
| lemline=lemline.lower() | |
| if not isSentence(lemline): | |
| continue | |
| lemmas=lemline.split() | |
| for lemma in lemmas: | |
| word=lemma.split("_")[0] | |
| pos=lemma.split("_")[-1] | |
| if (not pos in string.punctuation) and pos!="sent" and pos!="sym": | |
| lemmaposlist.append(lemma) | |
| lemmalist.append(word) | |
| wordtokens+=1 | |
| wordtypes[word]=1 | |
| if (not word in wordranks[-2000:]) and pos != "cd": | |
| swordtypes[word]=1 | |
| swordtokens+=1 | |
| if pos[0]=="n": | |
| lextypes[word]=1 | |
| nountypes[word]=1 | |
| lextokens+=1 | |
| nountokens+=1 | |
| if not word in wordranks[-2000:]: | |
| slextypes[word]=1 | |
| slextokens+=1 | |
| elif pos[0]=="j": | |
| lextypes[word]=1 | |
| adjtypes[word]=1 | |
| lextokens+=1 | |
| adjtokens+=1 | |
| if not word in wordranks[-2000:]: | |
| slextypes[word]=1 | |
| slextokens+=1 | |
| elif pos[0]=="r" and (adjdict.has_key(word) or (word[-2:]=="ly" and adjdict.has_key(word[:-2]))): | |
| lextypes[word]=1 | |
| advtypes[word]=1 | |
| lextokens+=1 | |
| advtokens+=1 | |
| if not word in wordranks[-2000:]: | |
| slextypes[word]=1 | |
| slextokens+=1 | |
| elif pos[0]=="v" and not word in ["be","have"]: | |
| verbtypes[word]=1 | |
| verbtokens+=1 | |
| lextypes[word]=1 | |
| lextokens+=1 | |
| if not word in wordranks[-2000:]: | |
| sverbtypes[word]=1 | |
| slextypes[word]=1 | |
| slextokens+=1 | |
| # 1. lexical density | |
| ld=division(lextokens,wordtokens) | |
| # 2. lexical sophistication | |
| # 2.1 lexical sophistication | |
| ls1=division(slextokens,lextokens) | |
| ls2=division(len(swordtypes.keys()),len(wordtypes.keys())) | |
| # 2.2 verb sophistication | |
| vs1=division(len(sverbtypes.keys()),verbtokens) | |
| vs2=division(len(sverbtypes.keys())*len(sverbtypes.keys()),verbtokens) | |
| cvs1=division(len(sverbtypes.keys()),sqrt(2*verbtokens)) | |
| # 3 lexical diversity or variation | |
| # 3.1 NDW, may adjust the values of "standard" | |
| ndw=ndwz=ndwerz=ndwesz=len(wordtypes.keys()) | |
| if len(lemmalist)>=standard: | |
| ndwz=getndwfirstz(standard,lemmalist) | |
| ndwerz=getndwerz(standard,lemmalist) | |
| ndwesz=getndwesz(standard,lemmalist) | |
| # 3.2 TTR | |
| ttr=msttr=division(len(wordtypes.keys()),wordtokens) | |
| if len(lemmalist)>=standard: | |
| msttr=getmsttr(standard,lemmalist) | |
| cttr=division(len(wordtypes.keys()),sqrt(2*wordtokens)) | |
| rttr=division(len(wordtypes.keys()),sqrt(wordtokens)) | |
| logttr=division(log(len(wordtypes.keys())),log(wordtokens)) | |
| uber=division((log(wordtokens,10)*log(wordtokens,10)),log(wordtokens/float(len(wordtypes.keys())),10)) | |
| # 3.3 verb diversity | |
| vv1=division(len(verbtypes.keys()),verbtokens) | |
| svv1=division(len(verbtypes.keys())*len(verbtypes.keys()),verbtokens) | |
| cvv1=division(len(verbtypes.keys()),sqrt(2*verbtokens)) | |
| # 3.4 lexical diversity | |
| lv=division(len(lextypes.keys()),lextokens) | |
| vv2=division(len(verbtypes.keys()),lextokens) | |
| nv=division(len(nountypes.keys()),nountokens) | |
| adjv=division(len(adjtypes.keys()),lextokens) | |
| advv=division(len(advtypes.keys()),lextokens) | |
| modv=division((len(advtypes.keys())+len(adjtypes.keys())),lextokens) | |
| output=filename | |
| for measure in [len(wordtypes.keys()), len(swordtypes.keys()), len(lextypes.keys()), len(slextypes.keys()), wordtokens, swordtokens, lextokens, slextokens, ld, ls1, ls2, vs1, vs2, cvs1, ndw, ndwz, ndwerz, ndwesz, ttr, msttr, cttr, rttr, logttr, uber, lv, vv1, svv1, cvv1, vv2, nv, adjv, advv, modv]: | |
| if type(measure)==type(0.0): | |
| measure="%.2f" % measure | |
| output+=", "+str(measure) | |
| print output | |