I have written a file to extract text from news articles. The program works as I want it to except it takes hours for it to write the output files. I have tried different buffering options as well as adding the data I want written to an array before writing that to the file. I have almost no experience programming so I might just be doing something really dumb.The main issue with the program is when I'm printing ngrams. d2 contains all of the items I want to print and it prints it based off of an index in a list. d2 is generated quickly, and only the writing is slow.import os, refrom collections import Counterfrom math import sqrtpath = os.getcwd() + /20_newsgroups#print pathdef findFiles(path):    Files = []    for root, dirs, files in os.walk(path):        for f in files:           #print (os.path.join(root , f).replace(\\,/))            Files.append(os.path.join(root , f).replace(\\,/))    return Filesdef cleanData():    data = findFiles(path)        remove = []    for entry in data:        f = open(entry,'r')        for line in f:            if line.lower().startswith(subject: re:) or line.lower().startswith(subject: re:):                #flag = 1                remove.append(entry)                break            elif line.lower().startswith(subject: ):                break                #print line                #for word in line.split():                                      #  if word == Re: or word == re::                        #print(Found Re:)                   #     remove.append(entry)                   #     break    #print len(data)    #print len(set(data))    #print len(remove)    #print len(set(remove))    data2 = list(set(data) - set(remove))    #print len(data2)        return set(data2) #6694d1 = {} #d1.setdefault('test',[]).append('test2') ##this allows duplicate entries.SetofUniqueWords = set()def readInWords():    data = cleanData()    for entry in data:        flag = 1        AfterLinesLine = False        f = open(entry,'r')               for line in f:            if line.lower().startswith(subject:):                  if flag ==1:                    for word in line.lower().split(subject: )[1].split():                                                                for stuff in re.sub(r'[\W_]+',' ',word).lower().split():                            if not re.match('^ *[0-9 ]*$',stuff):                                d1.setdefault(entry,[]).append(stuff)                                SetofUniqueWords.add(stuff)                flag = 0 ##this is because multiple lines in a file can start with subject so now we stop after the first one                                           if AfterLinesLine == True:                for word in line.lower().split():                     for stuff in re.sub(r'[\W_]+',' ',word).lower().split():##                        if not re.match('^ *[0-9 ]*$',stuff):                            d1.setdefault(entry,[]).append(stuff)                            SetofUniqueWords.add(stuff)            if line.startswith(Lines:):                AfterLinesLine = TruereadInWords()ItemList = list(SetofUniqueWords)ObjID = {}def SetObjectIDs():    id = 0    for key in d1:        ObjID[key] = id        id = id + 1def printBagOWords():    f = open('bag.csv','w')    for key in d1:        #f.write(str(d1[key]))        words = d1[key]        freqs = Counter(words)        mag = 0        for i in freqs:            mag = mag + freqs[i]*freqs[i]        mag = sqrt(mag)        for item in freqs:            index = ItemList.index(item)            #print key.split(/)[-1],index,freqs[item]            f.write(str(ObjID[key])+','+str(index)+','+str(freqs[item]/mag)+'\n')            #f.write(key.split(/)[-1]+','+str(item)+','+str(freqs[item])+'\n')    f.close()    h = open('bag.clabel','w')    for entry in ItemList:        h.write(entry+'\n')    h.close()def printClassFilerlabel():    f = open('newsgroup.rlabel','w')    for key in d1:        f.write(str(ObjID[key])+','+key.split(/)[-2]+'_'+key.split(/)[-1]+'\n')    f.close()def printClassFile():    f = open('newsgroup.class','w')    for key in d1:        f.write(str(ObjID[key])+','+key.split(/)[-2]+'\n')    f.close()def printNGrams(n):       d2 = {}    SetofNGram = set()    for key in d1:        Str =          Str = Str.join(d1[key])        for i in range(0,len(Str)-n+1):            item = Str[i:i+n]            #if not re.match('^ *[0-9 ]*$',item):            SetofNGram.add(item)            d2.setdefault(key,[]).append(item)            #print Str[i:i+n]        #print len(Str)    IL = list(SetofNGram)    #print len(IL)    #count = 0    #print Done finding dictionaries    g = open('char'+ str(n) +'.csv','w')    for key in d1: #this is intentionally d1 and not d2 to make sure the files are written in the same order and they have the same keys so doesnt matter        #count = count + 1        #print count                words = d2[key]        freqs = Counter(words)        mag = 0        a = str(ObjID[key])        for i in freqs:            mag = mag + freqs[i]**2 #making unit vectors!!         mag = sqrt(mag)        for item in freqs:            index = IL.index(item)            g.write(a+','+str(index)+','+str(freqs[item]/mag)+'\n')    g.close()    h = open('char'+ str(n) +'.clabel','w')    for entry in IL:        h.write(entry+'\n')    h.close()SetObjectIDs()printBagOWords()     printNGrams(3)printNGrams(5)printNGrams(7)printClassFile() printClassFilerlabel()
Extract text from news articles
python;performance;beginner;python 2.7;file
A likely cause for the performance issues is the line index = IL.index(item) in printNGrams, because it does a linear search through the list of all ngrams. Use a dictionary that maps ngrams to indexes instead of the list. In fact, an OrderedDict can replace both of the variables SetOfNGram and IL.Other comments:Use with to open and automatically close files.Use string formatting instead of eg. 'char'+ str(n) +'.csv'Use more descriptive variable names.Prefer to pass data in function arguments instead of global variables.My rewrite of printNGrams. I would also change names of d1 and d2 but I'm not sure to what as I did not study the rest of the code.def printNGrams(n, d1):       d2 = {}    all_ngrams = collections.OrderedDict()    for key, value in d1.iteritems():        text =  .join(value)        for i in range(0,len(text)-n+1):            ngram = text[i:i+n]            all_ngrams.setdefault(ngram, len(all_ngrams))            d2.setdefault(key,[]).append(ngram)    with open('char{}.csv'.format(n), 'w') as g:        for key in d1:             obj_id = ObjID[key]            ngrams = d2[key]            freqs = Counter(ngrams)            mag = sqrt(sum(freq**2 for freq in freqs.itervalues()))            for ngram, freq in freqs.iteritems():                index = all_ngrams[ngram]                g.write('{},{},{}\n'.format(obj_id, index, freq/mag))    with open('char{}.clabel'.format(n), 'w') as h:        for ngram in all_ngrams:            h.write(ngram+'\n')