<jupyter_start><jupyter_text>Import libs<jupyter_code>from src.DataWrangle import FetchData
from src.BaseClasses import ConfigFile as CF, Helper as H

from src.Plot import Delta, PlotHelper as PH
from src.Plot import Hazard, Hazard_Wrapper as HW
from src.Plot import RateNumErr

import pandas as pd
import matplotlib.pyplot as plt

from imp import reload
reload(FetchData); reload(CF); reload(H);
reload(PH); reload(HW); reload(Hazard);  reload(Delta)
reload(RateNumErr)<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>feeds = pd.read_csv(CF.fname_feedback, encoding = "ISO-8859-1")
df_pair = pd.read_csv(CF.fname_pair, encoding = "ISO-8859-1")
df_occur = pd.read_csv(CF.fname_occur, encoding = "ISO-8859-1")
#Plot params
plt.rc('font', family='serif')
plt.rcParams.update({'font.size': 10})
plt.rc('xtick', labelsize='x-small')
plt.rc('ytick', labelsize='x-small')<jupyter_output><empty_output><jupyter_text>Plots: Survival Curves for Attempts<jupyter_code>errSets=['3;', '5; 6;', '5;', '7;']
dfs = HW.getDF_dict(df_pair, errSets=errSets)
maxX = 5<jupyter_output><empty_output><jupyter_text>Across Events<jupyter_code>params = {'fpath':CF.path_hazard, 'fname':'Survival Attempts Delta',
          'xlim':maxX, 'width':4, 'height':3, 
          'nrows':2, 'ncols':2, 'height_ratios':[2,1],
          'wspace':0.1, 'hspace':0.25, 'revSortSplit':True,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1,
           'splitKeys':['labs', 'exam']}
fig, gs, kmfs = HW.plot_indiv(dfs, key='all',  Tname='numAttempt', groupBy='tool', splitBy='event', params=params)

params = {'fpath':CF.path_hazard, 'fname':'Survival Attempts Delta',
          'minX':0, 'maxX':maxX, 'minY':-0.1, 'maxY':0.1,
          'freq':1, 'newFig':False, 'ncols':2, 
          'showXTicks':[3,4], 'showYTicks':[1,3],
          'xlabels':['', '', '', ''],
          'ylabels':['', 'actual', '', r'$\Delta$ avg'],
          'xsuplabel':r'$\#$ Attempts', 'xsuppad': 0.05,
          'ysuplabel':'Survival Probability', 'ysuppad':0.02,
          'left':0.13, 'right':0.95,
          'bottom':0.18, 'top':0.82,
          'kmfs':kmfs
         }
fig = Delta.delta(fig, gs, params)<jupyter_output>key=all, TName=numAttempt, groupBy=tool, splitBy=event
	Survival
		['labs', 'exam']<jupyter_text>Across Individual Labs<jupyter_code>params = {'fpath':CF.path_hazard, 'fname':'Survival Attempts Lab Delta',
          'xlim':maxX, 'width':8, 'height':3, 
          'nrows':2, 'ncols':7, 'height_ratios':[2,1],
          'wspace':0.2, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1,
         'splitKeys':['LAB-01', 'LAB-02', 'LAB-03', 'LAB-04', 'LAB-05', 'LAB-06', 'LAB-07']}
fig, gs, kmfs = HW.plot_indiv(dfs, key='labs_1-6', Tname='numAttempt', groupBy='tool', splitBy='labs', params=params)

params = {'fpath':CF.path_hazard, 'fname':'Survival Attempts Lab Delta',
          'minX':0, 'maxX':maxX, 'minY':-0.15, 'maxY':0.15,
          'freq':1, 'newFig':False, 'ncols':7, 
          'showXTicks':range(8,15), 'showYTicks':[1,8],
          'ylabels':['']*6 + ['actual'] + ['']*6 + [r'$\Delta$ avg'],
          'xsuplabel':r'$\#$ Attempts', 'xsuppad': 0.02,
          'ysuplabel':'Survival Probability', 'ysuppad':0.02,
          'left':0.08, 'right':0.97,
          'bottom':0.15, 'top':0.82,
          'kmfs':kmfs
         }
fig = Delta.delta(fig, gs, params)<jupyter_output>key=labs_1-6, TName=numAttempt, groupBy=tool, splitBy=labs
	Survival
		['LAB-01', 'LAB-02', 'LAB-03', 'LAB-04', 'LAB-05', 'LAB-06', 'LAB-07']<jupyter_text>Across Selected Errors<jupyter_code>params = {'fpath':CF.path_hazard, 'fname':'Survival Attempts Errors Delta',
          'xlim':maxX, 'width':4, 'height':3, 
          'nrows':2, 'ncols':4, 'height_ratios':[2,1],
          'wspace':0.2, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1,
          'splitKeys':errSets,
          'replaceSplit':[r'$EG_{1}$', r'$EG_{7}$', r'$EG_{10}$', r'$EG_{19}$']}
fig, gs, kmfs = HW.plot_indiv(dfs, key='chosen-errs labs_1-6', Tname='numAttempt', groupBy='tool', splitBy='errSet', params=params)

params = {'fpath':CF.path_hazard, 'fname':'Survival Attempts Errors Delta',
          'minX':0, 'maxX':maxX, 'minY':-0.5, 'maxY':0.5,
          'freq':1, 'newFig':False, 'ncols':4, 
          'showXTicks':range(5, 9), 'showYTicks':[1,5],
          'ylabels':['']*3 + ['actual'] + ['']*3 + [r'$\Delta$ avg'],
          'xsuplabel':r'$\#$ Attempts', 'xsuppad': 0.02,
          'ysuplabel':'Survival Probability', 'ysuppad':0.02,
          'left':0.12, 'right':0.95,
          'bottom':0.15, 'top':0.82,
          'kmfs':kmfs
         }

fig = Delta.delta(fig, gs, params)<jupyter_output>key=chosen-errs labs_1-6, TName=numAttempt, groupBy=tool, splitBy=errSet
	Survival
		['3;', '5; 6;', '5;', '7;']<jupyter_text>Plots: Survival Curves for TimeTaken<jupyter_code>errSets=['3;', '5; 6;', '5;', '7;']
dfs = HW.getDF_dict(df_pair, errSets=errSets)
maxX = 200<jupyter_output><empty_output><jupyter_text>Across Events<jupyter_code>params = {'fpath':CF.path_hazard, 'fname':'Survival Time Delta',
          'xlim':maxX, 'width':4, 'height':3, 
          'nrows':2, 'ncols':2, 'height_ratios':[2,1],
          'wspace':0.1, 'hspace':0.25, 'revSortSplit':True,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1}
fig, gs, kmfs = HW.plot_indiv(dfs, key='all',  Tname='timeTaken (sec)', groupBy='tool', splitBy='event', params=params)

params = {'fpath':CF.path_hazard, 'fname':'Survival Time Delta',
          'minX':0, 'maxX':maxX, 'minY':-0.1, 'maxY':0.1,
          'freq':1, 'newFig':False, 'ncols':2, 
          'showXTicks':[3,4], 'showYTicks':[1,3],
          'xlabels':['']*4,
          'ylabels':['', 'actual', '', r'$\Delta$ avg'],
          'xsuplabel':r'Time-Taken (seconds)', 'xsuppad': 0.05,
          'ysuplabel':'Survival Probability', 'ysuppad':0.02,
          'left':0.13, 'right':0.95,
          'bottom':0.18, 'top':0.82,
          'kmfs':kmfs
         }
fig = Delta.delta(fig, gs, params)<jupyter_output>key=all, TName=timeTaken (sec), groupBy=tool, splitBy=event
	Survival
		['labs', 'exam']<jupyter_text>Across Individual Labs<jupyter_code>params = {'fpath':CF.path_hazard, 'fname':'Survival Time Lab Delta',
          'xlim':maxX, 'width':8, 'height':3, 
          'nrows':2, 'ncols':7, 'height_ratios':[2,1],
          'wspace':0.2, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1,
         'splitKeys':['LAB-01', 'LAB-02', 'LAB-03', 'LAB-04', 'LAB-05', 'LAB-06', 'LAB-07']}
fig, gs, kmfs = HW.plot_indiv(dfs, key='labs_1-6', Tname='timeTaken (sec)', groupBy='tool', splitBy='labs', params=params)

params = {'fpath':CF.path_hazard, 'fname':'Survival Time Lab Delta',
          'minX':0, 'maxX':maxX, 'minY':-0.25, 'maxY':0.25,
          'freq':1, 'newFig':False, 'ncols':7, 
          'showXTicks':range(8,15), 'showYTicks':[1,8],
          'ylabels':['']*6 + [r'$\delta$ best'] + ['']*6 + [r'$\Delta$ avg'],
          'xsuplabel':r'Time-Taken (seconds)', 'xsuppad': 0.02,
          'ysuplabel':'Survival Probability', 'ysuppad':0.02,
          'left':0.08, 'right':0.97,
          'bottom':0.15, 'top':0.82,
          'kmfs':kmfs
         }
fig = Delta.delta(fig, gs, params)<jupyter_output>key=labs_1-6, TName=timeTaken (sec), groupBy=tool, splitBy=labs
	Survival
		['LAB-01', 'LAB-02', 'LAB-03', 'LAB-04', 'LAB-05', 'LAB-06', 'LAB-07']<jupyter_text>Across Selected Errors<jupyter_code>params = {'fpath':CF.path_hazard, 'fname':'Survival Time Errors Delta',
          'xlim':maxX, 'width':4, 'height':3, 
          'nrows':2, 'ncols':4, 'height_ratios':[2,1],
          'wspace':0.2, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1,
          'splitKeys':errSets,
          'replaceSplit':[r'$EG_{1}$', r'$EG_{7}$', r'$EG_{10}$', r'$EG_{19}$']
         }
fig, gs, kmfs = HW.plot_indiv(dfs, key='chosen-errs labs_1-6', Tname='timeTaken (sec)', groupBy='tool', splitBy='errSet', params=params)

params = {'fpath':CF.path_hazard, 'fname':'Survival Time Errors Delta',
          'minX':0, 'maxX':maxX, 'minY':-0.5, 'maxY':0.5,
          'freq':1, 'newFig':False, 'ncols':4, 
          'showXTicks':range(5, 9), 'showYTicks':[1,5],
          'ylabels':['']*3 + ['actual'] + ['']*3 + [r'$\Delta$ avg'],
          'xsuplabel':r'Time-Taken (seconds)', 'xsuppad': 0.02,
          'ysuplabel':'Survival Probability', 'ysuppad':0.02,
          'left':0.12, 'right':0.95,
          'bottom':0.15, 'top':0.82,
          'kmfs':kmfs
         }
fig = Delta.delta(fig, gs, params)<jupyter_output>key=chosen-errs labs_1-6, TName=timeTaken (sec), groupBy=tool, splitBy=errSet
	Survival
		['3;', '5; 6;', '5;', '7;']<jupyter_text>Plots: Rate of Change  Errors<jupyter_code>y_err = 'numErrors'
yType_err = 'numErrors_LOC-sumWindow'
CF.rate_winSize = 30
CF.rate_winSkip = 15

df_fail_err, df_pass_err = RateNumErr.get_dfFail(df_occur) 
dict_dfNumErr_err = RateNumErr.get_dict_dfNumErr(df_fail_err, df_pass_err, y_err)

# Multiply by 100 to get per 100 LOC
dict_dfNumErr_err['ALL'][yType_err] = dict_dfNumErr_err['ALL'][yType_err]*100
params = {'fpath':CF.path_rateNumErr, 'fname':'Rate numErrors',
          'y':y_err, 'yType': yType_err, 
          'width':8, 'height':3, 
          'nrows':2, 'ncols':7, 'height_ratios':[2,1],
          'minX':13.75, 'maxX':17.25,
          'wspace':0.1, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1}

fig, gs, points = RateNumErr.plot_bar(dict_dfNumErr_err, params)

params = {'fpath':CF.path_rateNumErr, 'fname':'Rate numErrors',
          'minX':13.75, 'maxX':17.25, #'minY':-0.25, 'maxY':0.25,
          'freq':1, 'newFig':False, 'ncols':7, 'markersize':2,
          'showXTicks':range(8,15), 'showYTicks':[1,8],
          'titles': ['LAB-0'+str(i) for i in range(1,8)] + ['']*2*7,
          'ylabels': ['']*6 + ['actual'] + ['']*6+[r'$\Delta$ avg'],
          'xsuplabel':r'Hour', 'xsuppad': 0.05,
          'ysuplabel':r'$\#$ Errors per 100 LOC', 'ysuppad':0.02,
          'left':0.08, 'right':0.97,
          'bottom':0.15, 'top':0.85,
          'points':points, 'xticks':[14, 15, 16, 17]
         }
fig = Delta.delta(fig, gs, params)<jupyter_output><empty_output><jupyter_text>Attempt<jupyter_code>reload(RateNumErr)

y_attempt = 'numAttempt'
Settings.rate_winSize = 30
Settings.rate_winSkip = 15

df_fail_attempt, df_pass_attempt = RateNumErr.get_dfFail(df_pair) 
dict_dfNumErr_attempt = RateNumErr.get_dict_dfNumErr(df_fail_attempt, df_pass_attempt, y_attempt)
reload(RateNumErr); reload(PH); reload(Delta)

params = {'fpath':Settings.path_rateNumErr, 'fname':'Rate numAttempt',
          'y':y_attempt, 'yType': y_attempt, 
          'minX':13.75, 'maxX':17.25,
          'width':8, 'height':3, 
          'nrows':2, 'ncols':7, 'height_ratios':[2,1],
          'wspace':0.1, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1}

fig, gs, points = RateNumErr.plot_bar(dict_dfNumErr_attempt, params)

params = {'fpath':Settings.path_rateNumErr, 'fname':'Rate numAttempt',
          'minX':13.75, 'maxX':17.25, #'minY':-0.25, 'maxY':0.25,
          'freq':1, 'newFig':False, 'ncols':7, 'markersize':2,
          'showXTicks':range(8,15), 'showYTicks':[1,8],
          'titles': ['LAB-0'+str(i) for i in range(1,8)] + ['']*2*7,
          'ylabels': ['']*6 + ['actual'] + ['']*6 + [r'$\Delta$ avg'],
          'xsuplabel':r'Hour', 'xsuppad': 0.05,
          'ysuplabel':r'$\#$ Attempts', 'ysuppad':0.02,
          'left':0.08, 'right':0.97,
          'bottom':0.15, 'top':0.85,
          'points':points, 'xticks':[14, 15, 16, 17]
         }
fig = Delta.delta(fig, gs, params)<jupyter_output><empty_output><jupyter_text>TimeTaken<jupyter_code>reload(RateNumErr)

y_time = 'timeTaken'
Settings.rate_winSize = 30
Settings.rate_winSkip = 15

df_fail_time, df_pass_time = RateNumErr.get_dfFail(df_pair) 
dict_dfNumErr_time = RateNumErr.get_dict_dfNumErr(df_fail_time, df_pass_time, y_time)
reload(RateNumErr); reload(PH); reload(Delta)

params = {'fpath':Settings.path_rateNumErr, 'fname':'Rate timeTaken',
          'y':y_time, 'yType': y_time, 
          'minX':13.75, 'maxX':17.25,
          'width':8, 'height':3, 
          'nrows':2, 'ncols':7, 'height_ratios':[2,1],
          'wspace':0.1, 'hspace':0.25,
          'top':0.75, 'left':0.08, 'bot':0.2, 'xlabelpad':6, 'ylabelpad':1}

fig, gs, points = RateNumErr.plot_bar(dict_dfNumErr_time, params)

params = {'fpath':Settings.path_rateNumErr, 'fname':'Rate timeTaken',
          'minX':13.75, 'maxX':17.25, #'minY':-0.25, 'maxY':0.25,
          'freq':1, 'newFig':False, 'ncols':7, 'markersize':2,
          'showXTicks':range(8,15), 'showYTicks':[1,8],
          'titles': ['LAB-0'+str(i) for i in range(1,8)] + ['']*2*7,
          'ylabels': ['']*6 + ['actual'] + ['']*6 + [r'$\Delta$ avg'],
          'xsuplabel':r'Hour', 'xsuppad': 0.05,
          'ysuplabel':r'Time-Taken (seconds)', 'ysuppad':0.02,
          'left':0.08, 'right':0.97,
          'bottom':0.15, 'top':0.85,
          'points':points, 'xticks':[14, 15, 16, 17]
         }
fig = Delta.delta(fig, gs, params)<jupyter_output><empty_output><jupyter_text>Errors Histogram<jupyter_code>reload(CompErrs_hist); reload(Settings)

fpath, fname = Settings.path_compErrs, 'compilation_histogram'
fig, errs_count = CompErrs_hist.plot_hist_ces(df_occur)

params = {'fpath':fpath, 'fname':fname,
          'minY':10**1, 'maxY':10**4,
          'xsuplabel':r'Compilation Error-Group (EG) ID', 'xsuppad': 0.03,
          'ysuplabel':r'#Buggy Programs', 'ysuppad':0.03,
          'left':0.15, 'right':0.94,
          'bottom':0.15, 'top':0.97,
         }

Delta.adjust_figs(fig, None, params)
PH.savefig(fig, fpath, fname, fname)<jupyter_output><empty_output><jupyter_text>Debug<jupyter_code># The top err-sets ordered by counts of df_occur
errs_count[:100]<jupyter_output><empty_output><jupyter_text>Errors Area Under Curve<jupyter_code>fig, errs_pair = CompErrs_hist.plot_hist_ces(df_pair)
fname = 'Survival Time Errors AUC'

topK = 100
fig = CompErrs_AUC.plot_auc(df_pair, errs_pair, topK)

params = {'fpath':Settings.path_compErrs, 'fname':fname,
          'minX':-40, 'minY':0, 'maxX':100, #'minY':-0.2, 'maxY':0.2,
          #'freq':1, 'newFig':False, 'ncols':9, 
          'title': 'Top-'+str(topK)+' Compiler-Errors',
          'showXTicks':[1], 'showYTicks':[1],
          #'ylabels':['AUC of Baseline Survival'], 'ylabelpos':'left',
          #'xlabels':[r'$\Delta$ AUC of Survival (Best_Feedback - Avg_Baseline)'],
          'xsuplabel':'Potential Performance Improvement', 'xsuppad': 0.02,
          'ysuplabel':'Potential Hardness', 'ysuppad':0.02,
          'left':0.12, 'right':0.96,
          'bottom':0.12, 'top':0.98,
         }  
    
#axes = fig.get_axes()
Delta.adjust_figs(fig, None, params)
PH.savefig(fig, Settings.path_compErrs, fname, fname)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2169471153846154, 'fraction_numerical': 0.05194978632478633, 'mean_word_length': 3.87373901724699, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '86646', 'license': 'Apache-2.0', 'n_tokens_mistral': 6642, 'n_tokens_neox': 6184, 'n_words': 962, 'path': 'plots.ipynb', 'repo_name': 'umairzahmed/seet2020', 'size': '14976'}