I am studying the techniques of data mining and data processing. I'm doing this through data I've collected and stored in a csv file. The problem is that this filed was very large, to the point of having as astonishing 40 thousand lines of text.Some of the algorithms in the rendering part are fast and agile, but the part of the orthographic correction of words is laborious. I am using the NLTK package nltk.corpus import forest. So when it comes time to do this step, I daresay it will not end in a timely manner.From this, I was wondering if someone can help me with a solution where I can read a file line, do the whole process, save it to the bank and then read another line from the file. So by reading line by line and each line do the process. I think this way I can improve the performance of the algorithm.txtCorpus = []dtype_dic= {'status_id': str, 'status_message' : str, 'status_published':str}for csvfile in pd.read_csv('data/MyCSV.csv',dtype=dtype_dic,encoding='utf-8',sep=',', header='infer',engine='c', chunksize=2):    txtCorpus.append(csvfile)def status_processing(txtCorpus):    myCorpus = preprocessing.PreProcessing()    myCorpus.text = str(txtCorpus)    print Doing the Initial Process...    myCorpus.initial_processing()    print Done.    print ----------------------------    print (StartingLexical Diversity...)    myCorpus.lexical_diversity()    print Done    print ----------------------------    print Removing Stopwords...    myCorpus.stopwords()    print Done    print ----------------------------    print Lemmatization...    myCorpus.lemmatization()    print Feito    print ----------------------------    print Correcting the words...    myCorpus.spell_correct()    print Done    print ----------------------------    print Untokenizing...    word_final = myCorpus.untokenizing()    print Feito    print ----------------------------    print Saving in DB....    try:        db.myDB.insert(word_final, continue_on_error=True)    except pymongo.errors.DuplicateKeyError:        pass    print Insertion in the BB Completed. End of the Pre-Processing Process def main():    status_processing(txtCorpus)main()I believe that by visualizing the code, you can better understand what I explained above. I thought about doing a for where I read a line and passed it on def status_processing(txtCorpus): and so, I repeated the process until the end. But I could not reach a solution.preprocessing file:import nltk,re, htmlentitydefsfrom nltk.stem.snowball import SnowballStemmerfrom nltk.stem import WordNetLemmatizerfrom bs4 import BeautifulSoupimport spellcorrectclass Techniques(object):    Lemmatizing  = 1    Stopwords  = 2    Stemming = 3    Spellcorrect = 4    def __init__(self, Type):        self.value = Type    def __str__(self):        if self.value == Techniques.Lemmatizing:            return 'Lemmatizing'        if self.value == Techniques.Stopwords:            return 'Stopwords'        if self.value == Techniques.Stemming:            return 'Stemming'        if self.value == Techniques.Spellcorrect:            return 'Spell Correct'    def __eq__(self,y):       return self.value==y.valueclass PreProcessing():    @property    def text(self):        return self.__text    @text.setter    def text(self, text):        self.__text = text    tokens = None    def initial_processing(self):        soup = BeautifulSoup(self.text,html.parser)        self.text = soup.get_text()        #Todo Se quiser salvar os links mudar aqui        self.text = re.sub(r'(http://|https://|www.)[^\' ]+',  , self.text)        self.tokens = self.tokenizing(1, self.text)        pass    def lexical_diversity(self):        word_count = len(self.text)        vocab_size = len(set(self.text))        return vocab_size / word_count    def tokenizing(self, type, text):        if (type == 1):            return nltk.tokenize.word_tokenize(text)        elif (type == 2):            stok = nltk.data.load('tokenizers/punkt/portuguese.pickle')            #stok = nltk.PunktSentenceTokenizer(train)            return stok.tokenize(text)    def stopwords(self):        stopwords = nltk.corpus.stopwords.words('portuguese')        stopWords = set(stopwords)        palavroesPortugues = ['foda','caralho', 'porra', 'puta', 'merda', 'cu', 'foder', 'viado', 'cacete']        stopWords.update(palavroesPortugues)        filteredWords = []        for word in self.tokens:            if word not in stopWords:                filteredWords.append(word)        self.tokens = filteredWords    def stemming(self):        snowball = SnowballStemmer('portuguese')        stemmedWords = []        for word in self.tokens:            stemmedWords.append(snowball.stem(word))        self.tokens = stemmedWords    def lemmatization(self):        lemmatizer = WordNetLemmatizer()#'portuguese'        lemmatizedWords = []        for word in self.tokens:            lemmatizedWords.append(lemmatizer.lemmatize(word, pos='v'))        self.tokens = lemmatizedWords    def part_of_speech_tagging(self):        return 'Not implemented yet'    def padronizacaoInternetes(self):        return 'Not implementes yet'    def untokenize(self, words):                Untokenizing a text undoes the tokenizing operation, restoring        punctuation and spaces to the places that people expect them to be.        Ideally, `untokenize(tokenize(text))` should be identical to `text`,        except for line breaks.                text = ' '.join(words)        step1 = text.replace(`` , '').replace( '', '').replace('. . .',  '...')        step2 = step1.replace( ( ,  ().replace( ) , ) )        step3 = re.sub(r' ([.,:;?!%]+)([ \'`])', r\1\2, step2)        step4 = re.sub(r' ([.,:;?!%]+)$', r\1, step3)        step5 = step4.replace( ', ').replace( n't, n't).replace(             can not, cannot)        step6 = step5.replace( ` ,  ')        return step6.strip()    def untokenizing(self):        return ' '.join(self.tokens)        #return self.untokenize(self.tokens)        #return tokenize.untokenize(self.tokens)    def spell_correct(self):        correctedWords = []        spell = spellcorrect.SpellCorrect()        for word in self.tokens:            correctedWords.append(spell.correct(word))        self.tokens = correctedWordsspellcorret file: import re, collectionsfrom nltk.corpus import florestaclass SpellCorrect:    def words(self, text): return re.findall('[a-z]+', text.lower())    def train(features):        model = collections.defaultdict(lambda: 1)        for f in features:            model[f] += 1        return model    NWORDS = train(floresta.words()) #words(file('big.txt').read())    alphabet = 'abcdefghijklmnopqrstuvwxyz'    def edits1(self, word):       splits     = [(word[:i], word[i:]) for i in range(len(word) + 1)]       deletes    = [a + b[1:] for a, b in splits if b]       transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b)>1]       replaces   = [a + c + b[1:] for a, b in splits for c in self.alphabet if b]       inserts    = [a + c + b     for a, b in splits for c in self.alphabet]       return set(deletes + transposes + replaces + inserts)    def known_edits2(self, word):        return set(e2 for e1 in self.edits1(word) for e2 in self.edits1(e1) if e2 in self.NWORDS)    def known(self, words): return set(w for w in words if w in self.NWORDS)    def correct(self, word):        candidates = self.known([word]) or self.known(self.edits1(word)) or self.known_edits2(word) or [word]        return max(candidates, key=self.NWORDS.get)
Read a CSV file and do natural language processing on the data
python;time limit exceeded;csv;natural language processing
spellcorrect.pySpellCorrect should not be a class. Your two working methods (train and edit1) does not reference self at all and the other ones only use self for its namespace. You should provide functions instead.As far as I can tell, the words method is not used anymore since you commented it in the building of NWORDS.alphabet is better imported from string: from string import ascii_lowercase as alphabet.I don't understand the definition of model in train. Why give a score of 1 to missing features; and so a score of 2 for features encountered once? Moreover, it the aim of train is to count how many times a given feature appears in features, you'd be better off using a collection.Counter.You will have a better memory footprint if you turn edit1 into a generator. Just yield (and yield from in Python 3) computed elements instead of storing them in a list.Turning edit1 into a generator will allow you to do so with edit2 without requiring it to filter elements itself. And let this job to known alone. Avoiding the discrepancy between how you build your words.In edit1, you can iterate more easily over words and still get the index using enumerate. It can simplify some of your checks.import collectionsfrom string import ascii_lowercase as alphabetfrom nltk.corpus import florestaNWORDS = collections.Counter(floresta.words())def edits1(word):   for i, letter in enumerate(word):       begin, end = word[:i], word[i+1:]       yield begin + end  # delete       if end:           yield begin + end[0] + letter + end[1:]  # transpose       else:           for other in alphabet:               yield begin + letter + other  # insert at the end       for other in alphabet:           yield begin + other + end  # replace           yield begin + other + letter + end  # insert before the current letterdef edits2(word):    for editted_once in edits1(word):        for editted_twice in edits1(editted_once):            yield editted_twicedef known(words):    return set(w for w in words if w in NWORDS)def correct(word):    candidates = known([word]) or known(edits1(word)) or known(edits2(word)) or [word]    return max(candidates, key=self.NWORDS.get)Main fileYour top-level code should be under an if __name__ == '__main__': clause. So move your txtCorpus building and your call to main there.In fact, main would be of better interest if it built txtCorpus itself before calling status_processing.status_processing also does more than it advertise as it process the statuses, but also save the result in DB. You should let the caller do whatever they want with the processed results.All these print can be unneeded to someone else. Consider using the logging module instead.def status_processing(corpus):    myCorpus = preprocessing.PreProcessing()    myCorpus.text = str(corpus)    print Doing the Initial Process...    myCorpus.initial_processing()    print Done.    print ----------------------------    print (StartingLexical Diversity...)    myCorpus.lexical_diversity()    print Done    print ----------------------------    print Removing Stopwords...    myCorpus.stopwords()    print Done    print ----------------------------    print Lemmatization...    myCorpus.lemmatization()    print Feito    print ----------------------------    print Correcting the words...    myCorpus.spell_correct()    print Done    print ----------------------------    print Untokenizing...    word_final = myCorpus.untokenizing()    print Feito    print ----------------------------    return word_finalif __name__ == '__main__':    dtype_dic = {'status_id': str, 'status_message': str, 'status_published': str}    txt_corpus = list(pd.read_csv(        'data/MyCSV.csv', dtype=dtype_dic,        encoding='utf-8', sep=',',        header='infer', engine='c', chunksize=2))    word_final = status_processing(txt_corpus)    print Saving in DB....    try:        db.myDB.insert(word_final, continue_on_error=True)    except pymongo.errors.DuplicateKeyError:        pass    print Insertion in the DB Completed. End of the Pre-Processing Process preprocessing.pyTechniques should be an enum. You can use flufl.enum if you need them in Python 2. But I don't see them used anywhere in the code, you can get rid of that class.Since it seems that the code is for Python 2, you should have PreProcessing inherits from object.The text property of PreProcessing does not add value over a self.text attribute initialized in the constructor. Especially since you need to set it for the other methods to work.pass is unnecessary for non-empty blocks.tokenizing offers a choice between two variants, a boolean parameter would be more suited here. And since you seems to use only one of them, you can give it a default value.I would merge __init__ and initial_processing since this method populate the self.tokens attribute with the initial set of tokens every other method works with.Using raise NotImplementedError instead of return 'Not implemented yet' is much more meaningfull.Consider using list-comprehensions or the list constructor instead of manually appending items into an empty list.import nltkimport refrom nltk.stem.snowball import SnowballStemmerfrom nltk.stem import WordNetLemmatizerfrom bs4 import BeautifulSoupimport spellcorrectclass PreProcessing():    def __init__(self, text):        soup = BeautifulSoup(text, html.parser)        #Todo Se quiser salvar os links mudar aqui        self.text = re.sub(r'(http://|https://|www.)[^\' ]+',  , soup.get_text())        self.tokens = self.tokenizing()    def lexical_diversity(self):        word_count = len(self.text)        vocab_size = len(set(self.text))        return vocab_size / word_count    def tokenizing(self, use_default_tokenizer=True):        if use_default_tokenizer:            return nltk.tokenize.word_tokenize(self.text)        stok = nltk.data.load('tokenizers/punkt/portuguese.pickle')        return stok.tokenize(self.text)    def stopwords(self):        stopwords = set(nltk.corpus.stopwords.words('portuguese'))        stopwords.update([            'foda', 'caralho', 'porra',            'puta', 'merda', 'cu',            'foder', 'viado', 'cacete'])        self.tokens = [word for word in self.tokens if word not in stopwords]    def stemming(self):        snowball = SnowballStemmer('portuguese')        self.tokens = [snowball.stem(word) for word in self.tokens]    def lemmatization(self):        lemmatizer = WordNetLemmatizer()  #'portuguese'        self.tokens = [lemmatizer.lemmatize(word, pos='v') for word in self.tokens]    def part_of_speech_tagging(self):        raise NotImplementedError    def padronizacaoInternetes(self):        raise NotImplementedError    def untokenize(self, words):                Untokenizing a text undoes the tokenizing operation, restoring        punctuation and spaces to the places that people expect them to be.        Ideally, `untokenize(tokenize(text))` should be identical to `text`,        except for line breaks.                text = ' '.join(words)        step1 = text.replace(`` , '').replace( '', '').replace('. . .',  '...')        step2 = step1.replace( ( ,  ().replace( ) , ) )        step3 = re.sub(r' ([.,:;?!%]+)([ \'`])', r\1\2, step2)        step4 = re.sub(r' ([.,:;?!%]+)$', r\1, step3)        step5 = step4.replace( ', ').replace( n't, n't).replace(             can not, cannot)        step6 = step5.replace( ` ,  ')        return step6.strip()    def untokenizing(self):        return ' '.join(self.tokens)    def spell_correct(self):        self.tokens = [spellcorrect.correct(word) for word in self.tokens]More generic commentsConsider reading (and following) PEP 8, the official Python style guide; especially as regard to:import declarations;whitespace around operators, comasand variable names.Also consider using docstrings all around your code, it will make it more easier to understand.