The program below is about finding the values below the percentage and splitting them into five different output files. This works great for small input data files. But when I am trying to run it for a big data it takes ages to compute. Any suggestions on optimizing the program so that I can get fast computation?#seperate by col 5import numpy as npimport osimport reimport mathimport csvimport sysimport timeimport multiprocessingfrom scipy import statscurdir = 'Tools/'inputdir = 'input/'filename = 'input'outputdir = 'output/'def seperate(filename):    print Begin seperating File: %s % filename    dtype = np.dtype([('id1',str,12),('id2',str,12),('c1',int),('c2',int),('c3',float)])    data = np.loadtxt(curdir + inputdir + filename,dtype=dtype)    col = 'c3'    col_data = np.array(data[:][col])    cache = {}    def get_per(per):            if(cache.has_key(per)):                return cache.get(per)            else:                r = 100 - stats.percentileofscore(col_data,per)                cache[per] = r                return r    o1 = file(curdir + outputdir+ filename + '_0_20','w')    o2 = file(curdir + outputdir+ filename + '_20_40','w')    o3 = file(curdir + outputdir+ filename + '_40_60','w')    o4 = file(curdir + outputdir+ filename + '_60_80','w')    o5 = file(curdir + outputdir+ filename + '_80_100','w')    for line in data:        per = get_per(line[col])        output_format = %s %s %d %d %.1f\n        output_data = (line['id1'],line['id2'],line['c1'],line['c2'],line['c3'])        if per>=0 and per < 20:            o1.write(output_format % output_data)        elif per>=20 and per<40:            o2.write(output_format % output_data)            pass        elif per>=40 and per<60:            o3.write(output_format % output_data)            pass        elif per>=60 and per<80:            o4.write(output_format % output_data)            pass        elif per>=80 and per<=100:            o5.write(output_format % output_data)            pass    o1.close()    o2.close()    o3.close()    o4.close()    o5.close()    print Finish seperating File: %s % filenameps = []print Reading Filesfor parent,dirNames,fileNames in os.walk(curdir+inputdir):    if fileNames:        #multi_process        print fileNames        for fileName in fileNames:            m = re.match(r'^(?!\.)',fileName)            if m:                ps.append(multiprocessing.Process(target=seperate,args=(fileName,)))#runing by multiple processesfor p in ps:    p.start()for p in ps:    p.join()
Splitting many data files by quintile
python;file;time limit exceeded;numpy;multiprocessing