<jupyter_start><jupyter_text>读取时延<jupyter_code>read_latency_row = pd.read_csv('eclogs/readLatency.txt', names = ['file', 'block', 'ms'], sep=' ')
for index in read_latency_row.index:
    if read_latency_row['ms'][index] > 1000:
        read_latency_row = read_latency_row.drop(index)
read_latency_row = read_latency_row.reset_index(drop=True)
read_latency_blk = read_latency_row.groupby(['file', 'block']).mean().unstack()
read_latency_blk
# read_latency_node_mean = read_latency_row.drop(columns=['block']).groupby(['file', 'node']).mean().unstack()
# read_latency_node_mean
block_location = pd.read_csv('eclogs/workersToWrite.txt',names = range(read_latency_blk.shape[-1]), sep = ' ')
read_latency_row['node'] = [block_location.at[read_latency_row.at[i, 'file'], read_latency_row.at[i, 'block']] for i in range(read_latency_row.shape[0])]
# read_latency_node = read_latency_row.drop('block', axis = 1).groupby(['file', 'node']).mean().unstack()
read_latency_node = pd.DataFrame(dict(list(read_latency_row.groupby('node')['ms'])))

read_latency_node_dic = dict()
for column in read_latency_node.columns:
    read_latency_node_dic[column] = read_latency_node[column].dropna().values

maxlen = 0
for key in read_latency_node_dic.keys():
    if key == 0:
        continue
    if len(read_latency_node_dic[key]) > maxlen:
        maxlen = len(read_latency_node_dic[key])

for key in read_latency_node_dic.keys():
    idx_to_drop = []
    if key == 0:
        continue
    for i in range(len(read_latency_node_dic[key])):
        if read_latency_node_dic[key][i] >= 1000:
            idx_to_drop.append(i)
    read_latency_node_dic[key] = np.delete(read_latency_node_dic[key], idx_to_drop)
# read_latency_node
for key in read_latency_node_dic.keys():
    line = ''
    if key == 0:
        continue
    for i in range(maxlen):
        line += str(read_latency_node_dic[key][i%len(read_latency_node_dic[key])]) + ' '
    print(line + ';')
# read_latency_node_dic
mean_time = read_latency_node.mean()
mean_time
fig, ax = plt.subplots(1, figsize = (10, 10))
ax.scatter(range(len(mean_time)), mean_time.values)
ax.set_title('mean latency on per node')<jupyter_output><empty_output><jupyter_text>节点服务能力统计<jupyter_code>size = [100, 200, 300, 400, 500, 600]
time = [[]]*4
time[0] = [105, 198, 282, 372, 464, 543]
time[1] = [112, 193, 279, 361, 474, 543]
time[2] = [68, 117, 173, 221, 271, 324]
time[3] = [109, 195, 281, 354, 434, 535]
plt.scatter(size, time[3])
[[]]*2<jupyter_output><empty_output><jupyter_text>解码时延<jupyter_code>decode_latency_row = pd.read_csv('eclogs/decodingLatency.txt', names = ['num', 'ms'], sep = ' ')<jupyter_output><empty_output><jupyter_text>按块数分组<jupyter_code>def yforx(x, y):
    dic = dict.fromkeys(set(x))
    length = list()
    for key in dic.keys():
        dic[key] = list()
    for idx in range(len(x)):
        dic[x[idx]].append(y[idx])
    for key in dic.keys():
        length.append(len(dic[key]))
    for key in dic.keys():
        dic[key].extend([None] * (max(length) - len(dic[key])))
    return dic
decode_latency = pd.DataFrame(yforx(decode_latency_row['num'], decode_latency_row['ms'])).astype('float64')
# print(decode_latency[1].dropna().values)
decode_latency_row
fig1, ax1 = plt.subplots(1, figsize = (15, 10))
ax1.scatter(decode_latency_row['num'], decode_latency_row['ms'])
ax1.set_title('decoding latency for number of blocks')
ax1.scatter(decode_latency_row['num'].unique(), decode_latency_row.groupby(['num']).mean(), color = 'r', s = 100)
plt.xlabel('decoding number')
plt.ylabel('decoding latency')
plt.savefig(fname = 'decoding latency.png', format = 'png')
from scipy.stats import chi2, f
chi2y = chi2.pdf(np.arange(0.001,50, 0.1), 10, loc = 5.3, scale = 0.3) * 0.11
fy =f.pdf(np.arange(0.001,50, 0.1), 4, 2)

fig, axes = plt.subplots(nrows = decode_latency.shape[1],ncols = 2, figsize = (20, 20), sharex = True, sharey = False)

if(decode_latency.shape[1] == 1):
    axes[0].scatter(range(len(decode_latency[i + 1])), decode_latency[i + 1].sort_values())
    axes[0].set_title('decoding latency rank for '+str(i+1)+' blocks')
    axes[1].set_title('decoding latency distribution for '+str(i+1)+' blocks')
    axes[1].hist(decode_latency[i + 1].dropna().values, bins = 50, density = 1)
else:
    for i in range(len(axes)): 
        axes[i, 0].scatter(range(len(decode_latency[i + 1])), decode_latency[i + 1].sort_values())
        axes[i, 0].set_title('decoding latency rank for '+str(i+1)+' blocks')
        axes[i, 1].set_title('decoding latency distribution for '+str(i+1)+' blocks')
        axes[i, 1].hist(decode_latency[i + 1].dropna().values, bins = 50, density = 1)
plt.xlabel('decoding latency')
plt.ylabel('frequency')
#     axes[i, 1].plot(chi2y)
plt.savefig(fname="解码时延2.png",format="png")<jupyter_output><empty_output><jupyter_text>从pi计算读取策略<jupyter_code>import pandas as pd
p = pd.read_csv('eclogs/p.txt',header = None, sep = '\s+')
lmp = pd.read_csv('eclogs/lmp.txt',header = None, sep = '\s+')
p
k = 3
C = 7
N = 4
# p['cache'] = k - p.sum(axis = 1)
data = []
for i in range(len(lmp)):
    data.extend(lmp.values[i])
def quick_sort(lists):
    if not lists:
        return []
    assert isinstance(lists, list)
    if len(lists) == 1:
        return lists
    pivot = lists.pop()
    llist, rlist = [], []
    for x in lists:
        if x>pivot:
            rlist.append(x)
        else:
            llist.append(x)
    return quick_sort(llist) + [pivot] + quick_sort(rlist)

def sort_topk(s, k):
    # return sorted(s)[:k]
    return quick_sort(s)[-k:]    

# def determine_palcement():
#     topk = sort_topk(data, C)
#     kth = topk[0]
#     for i in range(lmp.shape[1]):
#         for j in range(lmp.shape[0]):
#             if lmp[i][j] >= kth:
# #                 p[i][j] = 0
#                 continue
# #     placement = (0.1 - p) < 0
#     placement = (p > 0)
#     print(p)
#     return placement
p
import math

workersToWrite = list()
# placement = determine_palcement()
for index in p.index:
#     vc = placement.loc[index].value_counts()
#     new_row = [0] * int(N - sum(np.ceil(p.loc[index] - 0.01)))
    new_row = [0] * int(N - sum(np.ceil(p.loc[index])))
    
    tmp = []
    cnt0 = (p.loc[index] == 0).astype(int).sum()
    kth = sort_topk(list(p.loc[index]), k - cnt0)[0]
    for i in range(p.shape[1]):
        if (p.loc[index][i] >= kth) & (p.loc[index][i] != 0):
            tmp.append(i+1)
    new_row.extend(tmp)
    
    tmp = []
    for i in range(p.shape[1]):
        if (p.loc[index][i] < kth) & (p.loc[index][i] != 0):
            tmp.append(i+1)
    new_row.extend(tmp)
    workersToWrite.append(new_row)
pd.DataFrame(workersToWrite).to_csv('eclogs/workersToWrite.txt', index=False, header=False, sep = ' ')
workersToWrite                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          
circle_times = 40
read_list = list()

while(len(read_list) < circle_times*len(p)):
    rand = pd.DataFrame(np.random.rand(p.shape[0], p.shape[1]))
    read_matrix = (rand - p) < 0
    
    for index in read_matrix.index:
        new_row = []
        while(len(new_row) != k):
            rand = pd.DataFrame(np.random.rand(p.shape[0], p.shape[1]))
            read_matrix = (rand - p) < 0
            vc = read_matrix.loc[index].value_counts()
            new_row = list(np.arange(workersToWrite[index].count(0)))
            tmp = []
            for j in range(0, p.shape[1]):
                if (read_matrix.loc[index][j] == True) and (j+1 in workersToWrite[index]):
                    tmp.append(workersToWrite[index].index(j+1))
            new_row.extend(tmp)
#             print(new_row)
        read_list.append(new_row)
pd.DataFrame(read_list).to_csv('eclogs/blocksToRead.txt', index=False, header=False, sep = ' ')
np.arange(workersToWrite[3].count(0))
# np.random.randint(0, len(t))
import time
time.time()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.06008736738878488, 'fraction_numerical': 0.01074262280467148, 'mean_word_length': 0.37840992872941753, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 732, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '536448', 'license': 'Apache-2.0', 'n_tokens_mistral': 3241, 'n_tokens_neox': 3045, 'n_words': 541, 'path': 'python/.ipynb_checkpoints/plot-checkpoint.ipynb', 'repo_name': 'wushuhan/Alluxio-DataPlacement', 'size': '22434'}