<jupyter_start><jupyter_text>gpu info<jupyter_code>gtx950 = DeviceInfo()
gtx950.sm_num = 6
gtx950.sharedmem_per_sm = 49152
gtx950.reg_per_sm = 65536
gtx950.maxthreads_per_sm = 2048<jupyter_output><empty_output><jupyter_text>2 stream info<jupyter_code># 10M for mem_mem : where the h2d between streams are overlapped
trace_file = 'trace_10M_s1.csv'
trace_file_2cke = 'trace_h2d_h2d_ovlp.csv'

df_trace = read_trace.trace2dataframe(trace_file) # read the trace to the dataframe
df_trace_2cke = read_trace.trace2dataframe(trace_file_2cke)
#df_trace
#cuda_timeline.plot_trace(df_trace)
df_trace_2cke
cuda_timeline.plot_trace(df_trace_2cke)<jupyter_output>/home/leiming/anaconda2/lib/python2.7/site-packages/matplotlib/axes/_base.py:1292: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
  if aspect == 'normal':
/home/leiming/anaconda2/lib/python2.7/site-packages/matplotlib/axes/_base.py:1297: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal
  elif aspect in ('equal', 'auto'):<jupyter_text>1cke - read trace and reset the timeline<jupyter_code>df_single_stream = read_trace.get_timing(df_trace)
df_single_stream
df_s1 = read_trace.reset_starting(df_single_stream)
df_s1<jupyter_output><empty_output><jupyter_text>2cke case<jupyter_code>df_2stream = read_trace.get_timing(df_trace_2cke)
df_2stream
tot_runtime = read_trace.getTotalRuntime(df_2stream)
print tot_runtime<jupyter_output>37.784463<jupyter_text>2 cke<jupyter_code>stream_num = 2

# find when to start the stream and update the starting pos for the trace
H2D_H2D_OVLP_TH = 3.158431

df_cke_list = cke.init_trace_list(df_s1, stream_num = stream_num, h2d_ovlp_th = H2D_H2D_OVLP_TH)
df_cke_list[0]
df_cke_list[1]<jupyter_output><empty_output><jupyter_text>sort<jupyter_code>df_all_api = cke.init_sort_api_with_extra_cols(df_cke_list)
df_all_api<jupyter_output><empty_output><jupyter_text>start algo<jupyter_code>count = 1
# break_count = 7

while not cke.AllDone(df_all_api):
    # pick two api to learn 
    df_all_api, r1, r2 = cke.PickTwo(df_all_api)
    
    if r1 == None and r2 == None:                          # go directly updating the last wake api
        df_all_api = cke.UpdateStream_lastapi(df_all_api)
    else:
        df_all_api = cke.StartNext_byType(df_all_api, [r1, r2])

        whichType = cke.CheckType(df_all_api, r1, r2) # check whether the same api
#         print whichType

        if whichType == None:
            df_all_api = cke.Predict_noConflict(df_all_api, r1, r2)
        elif whichType in ['h2d', 'd2h']: # data transfer in the same direction
            df_all_api = cke.Predict_transferOvlp(df_all_api, r1, r2, ways = 2.0)
        else: # concurrent kernel: todo
            pass

    #     if count == break_count:
    #         break

        rangeT = cke.Get_pred_range(df_all_api)
#         print rangeT

    #     if count == break_count:
    #         break

        extra_conc = cke.Check_cc_by_time(df_all_api, rangeT) # check whether there is conc during the rangeT

        if extra_conc == 0:
            if whichType in ['h2d', 'd2h']:
                df_all_api = cke.Update_wake_transferOvlp(df_all_api, rangeT, ways = 2.0)
            elif whichType == 'kern':
                pass
            else: # no overlapping
                df_all_api = cke.Update_wake_noConflict(df_all_api, rangeT)

            # check if any api is done, and update the timing for the other apis in that stream
            df_all_api = cke.UpdateStreamTime(df_all_api)

        else: # todo : when there is additional overlapping
            pass

#         if count == break_count:
#             break
        
    # next call
    count = count + 1
df_all_api
df_all_api.loc[df_all_api.stream_id == 0]
df_all_api.loc[df_all_api.stream_id == 1]
#
# run above
#<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0840764331210191, 'fraction_numerical': 0.030318471337579617, 'mean_word_length': 3.7530266343825667, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '4240', 'license': 'MIT', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1434, 'n_words': 294, 'path': 'mem_mem/tests/2cke_transOvlp/2cke_transOvlp.ipynb', 'repo_name': '3upperm2n/trans_kernel_model', 'size': '3925'}