I have a big log file (say 1-3 Gb) which I need to parse, extract data & save it in a CSV file.Text File Data  *    D:40035FC8 wr-long         00000008 \\core0\Global\u4TimeHiCnt         1.000us  *    D:40027C5C rd-byte               00 *core0\Global\Ypf_OILL_OilLvlOn   20.342us  *    D:40010044 rd-word             0FE2 *l\u2SAD_OILLVS_RecoveryCounter    0.160us  *    D:40010044 wr-word             0FE1 *l\u2SAD_OILLVS_RecoveryCounter    0.040us  *    D:40035FC8 wr-long         00000008 \\core0\Global\u4TimeHiCnt         1.000usI have to extract the variable name which is after the last \ and then the number of Read & Write along with the datatype & store it in a CSV file.CSV File ResultVariable        Datatype     CORE 0      CORE 1      CORE X                           Read   Write   Read    Write   Read    Write  OS_inKernel         byte   0     0      111768 111878     0    0 OS_globalIntLevel   long   0     0      281604 237901     0    0The problem is it takes too much time. Can you please look at the attached code and suggest ways to make it faster?import stringimport sysimport timeMyFile = open(C:\\Users\\AEC_FULL\\Saravanan\\Workspace\\Trace32Log_Parser\\core1_sram_ReadWrite.txt)#core0_sram_ReadWrite_rawdataGeneratedFile = open(str((C:\\Users\\AEC_FULL\\Saravanan\\Workspace\\Trace32Log_Parser\\)+'ParsedOutput.csv'),'w')try:MyVariableList = []TimeStartTest       = time.time()        #Starting Time     GeneratedFile.write('\nVariable')GeneratedFile.write(', Datatype')GeneratedFile.write(', CORE 0')GeneratedFile.write(',, CORE 1')GeneratedFile.write(',, CORE X')GeneratedFile.write('\n,, Read ')GeneratedFile.write(', Write ')GeneratedFile.write(', Read ')GeneratedFile.write(', Write ')GeneratedFile.write(', Read ')GeneratedFile.write(', Write ')                            GeneratedFile.write('\n')      for CurrentLine in MyFile:    NoofSpaces = 0    if CurrentLine.find('\\') != -1:        MyVariable     = CurrentLine[CurrentLine.rfind('\\')+1:].split(' ')[0]                 elif CurrentLine.find('*\\') != -1:        MyVariable     = CurrentLine[CurrentLine.rfind('*\\')+1:].split(' ')[0]                 elif CurrentLine.find('*') != -1:                        MyVariable     = CurrentLine[CurrentLine.rfind('*')+1:].split(' ')[0]        VariableFound = 0                MyVariableList.sort()    Lowerbound = 0    Upperbound = len(MyVariableList)-1    while Lowerbound <= Upperbound and VariableFound == 0:        middle_pos = (Lowerbound+Upperbound) // 2        if MyVariableList[middle_pos] < MyVariable:            Lowerbound = middle_pos + 1        elif MyVariableList[middle_pos] > MyVariable:            Upperbound = middle_pos - 1        else:            VariableFound = 1            if VariableFound == 0:                    MyVariableList.append(MyVariable)         try:            MyFile1 = open(C:\\Users\\AEC_FULL\\Saravanan\\Workspace\\Trace32Log_Parser\\core1_sram_ReadWrite.txt)#core0_sram_ReadWrite_rawdata            Core0_ReadCount  = 0            Core0_WriteCount = 0            Core1_ReadCount  = 0            Core1_WriteCount = 0            CoreX_ReadCount  = 0            CoreX_WriteCount = 0                                for CurrentLine1 in MyFile1:                if CurrentLine1.find(MyVariable) != -1:                    ##   CORE 0  ##                    if CurrentLine1.find(0\\Global) != -1:                        DataType        = CurrentLine1.split('         ')[0].split('-')[1]                        DataOperation   = CurrentLine1.split('         ')[0].split('-')[0].split(' ')[-1]                        if DataOperation == 'rd':                            Core0_ReadCount = Core0_ReadCount + 1                        elif DataOperation == 'wr':                            Core0_WriteCount = Core0_WriteCount + 1                                                                  ##   CORE 1  ##                                            elif CurrentLine1.find(1\\Global) != -1:                                                        DataType        = CurrentLine1.split('         ')[0].split('-')[1]                        DataOperation   = CurrentLine1.split('         ')[0].split('-')[0].split(' ')[-1]                        if DataOperation == 'rd':                            Core1_ReadCount = Core1_ReadCount + 1                        elif DataOperation == 'wr':                            Core1_WriteCount = Core1_WriteCount + 1                                                ##   CORE X  ##                                            else:                        DataType        = CurrentLine1.split('         ')[0].split('-')[1]                        DataOperation   = CurrentLine1.split('         ')[0].split('-')[0].split(' ')[-1]                        if DataOperation == 'rd':                            CoreX_ReadCount =  CoreX_ReadCount + 1                        elif DataOperation == 'wr':                            CoreX_WriteCount = CoreX_WriteCount + 1            GeneratedFile.write('\n %s' %MyVariable)            GeneratedFile.write(', %s' %DataType)                                        GeneratedFile.write(', %d' %Core0_ReadCount)            GeneratedFile.write(', %d' %Core0_WriteCount)                         GeneratedFile.write(', %d' %Core1_ReadCount)            GeneratedFile.write(', %d' %Core1_WriteCount)              GeneratedFile.write(', %d' %CoreX_ReadCount)            GeneratedFile.write(', %d' %CoreX_WriteCount)                                        GeneratedFile.write('\n')                                                            finally:            MyFile1.close()            except:    print sys.exc_info()       finally:    GeneratedFile.close()      MyFile.close()    TimeStopTest       = time.time()    print str(int((TimeStopTest - TimeStartTest)/60))
Parsing a big text file, extract data & store it in a CSV file
python;performance;parsing;file;csv
StrategyThe cause of the performance problem is that for every line where you encounter a new variable name, you open the same file again and scan through, looking at every line that contains the same variable name.  That is a process that takes \$O(n^2)\$ time, where \$n\$ is the number of lines in the input file.  For large input files, anything slower than \$O(n)\$ would be unacceptable.A root cause of your problem is that you aren't using data structures effectively.  For example, you maintain a MyVariableList array, which you periodically sort so that you can do a binary search on it.  What you want is a set, or if you want to preserve the order of appearance of the variables in the output, an OrderedDict.A reasonable approach would make just one pass through the file, accumulating statistics in a data structure as you go, and then write a summary of that data structure when you reach the end of the input.The outline of your program should look like this.  To note:Organize code into functionsUse with blocks to open files, so that they will automatically be closedTake advantage of Python's csv library.from collections import OrderedDictimport csvimport redef analyze_log(f):    stats = OrderedDict()    for line in f:        re.search()            return statsdef write_stats(stats, f):    out = csv.writer(f)    out.writerow()    for var in stats:        out.writerow()def main(input_filename, output_filename):    with open(input_filename) as input_file:        stats = analyze_log(input_file)    with open(output_filename, 'w') as output_file:        write_stats(stats, output_file)if __name__ == '__main__':    main(r'C:\Users\AEC_FULL\\core1_sram_ReadWrite.txt',         r'C:\Users\AEC_FULL\\ParsedOutput.csv')Even better, take the input from fileinput.input(), and write the output to sys.stdout.  Use the shell command to specify the input files and redirect the output to a file, and avoid hard-coding the input and output filenames in your script.  Then you could just writeimport fileinputimport sysif __name__ == '__main__':    write_stats(analyze_log(fileinput.input()), sys.stdout)ParsingSlicing each line using finds and splits all over the place is confusing.  You would be much better off trying to make sense of the input file fields, then using regular expression matches.def analyze_log(f):    stats = OrderedDict()    for line in f:        _, _, rw_datatype, _, core_varname, _ = line.split()        match = re.search(r'.*[*\\](.*)', core_varname)        if not match:            continue        var = match.group(1)        match = re.search(r'([01])\\Global', core_varname)        core = match and match.group(1) or 'X'        rw, datatype = rw_datatype.split('-', 1)        var_stats = stats.get(var, {'rd': {'0': 0, '1': 0, 'X': 0},                                    'wr': {'0': 0, '1': 0, 'X': 0},                                    'type': datatype })        stats[var] = var_stats        var_stats[rw][core] += 1    return statsWritingUsing the CSV library would result in tidier code than writing commas and newlines.def write_stats(stats, f):    out = csv.writer(f)    out.writerow(['Variable', 'Datatype',                  'CORE 0', None, 'CORE 1', None, 'CORE X', None])    out.writerow([None, None] +                 ['Read', 'Write'] * 3)    for var in stats:        out.writerow([var, stats[var]['type'],                      stats[var]['rd']['0'], stats[var]['wr']['0'],                      stats[var]['rd']['1'], stats[var]['wr']['1'],                      stats[var]['rd']['X'], stats[var]['wr']['X']])