<jupyter_start><jupyter_text>从pi计算读取策略<jupyter_code>import pandas as pd
import numpy as np
p = pd.read_csv('eclogs/p.txt',header = None, sep = '\s+')
# lmp = pd.read_csv('eclogs/lmp.txt',header = None, sep = '\s+')
p
k = 3
C = 7
N = 4
# p['cache'] = k - p.sum(axis = 1)
# data = []
# for i in range(len(lmp)):
#     data.extend(lmp.values[i])
def quick_sort(lists):
    if not lists:
        return []
    assert isinstance(lists, list)
    if len(lists) == 1:
        return lists
    pivot = lists.pop()
    llist, rlist = [], []
    for x in lists:
        if x>pivot:
            rlist.append(x)
        else:
            llist.append(x)
    return quick_sort(llist) + [pivot] + quick_sort(rlist)

def sort_topk(s, k):
    # return sorted(s)[:k]
    return quick_sort(s)[-k:]    

# def determine_palcement():
#     topk = sort_topk(data, C)
#     kth = topk[0]
#     for i in range(lmp.shape[1]):
#         for j in range(lmp.shape[0]):
#             if lmp[i][j] >= kth:
# #                 p[i][j] = 0
#                 continue
# #     placement = (0.1 - p) < 0
#     placement = (p > 0)
#     print(p)
#     return placement<jupyter_output><empty_output><jupyter_text>生成写入分布策略<jupyter_code>import math

workersToWrite = list()
# placement = determine_palcement()
for index in p.index:
#     vc = placement.loc[index].value_counts()
#     new_row = [0] * int(N - sum(np.ceil(p.loc[index] - 0.01)))
    new_row = [0] * int(N - sum(np.ceil(p.loc[index])))
    
    tmp = []
    cnt0 = (p.loc[index] == 0).astype(int).sum()
    kth = sort_topk(list(p.loc[index]), k - cnt0)[0]
    for i in range(p.shape[1]):
        if (p.loc[index][i] >= kth) & (p.loc[index][i] != 0):
            tmp.append(i+1)
    new_row.extend(tmp)
    
    tmp = []
    for i in range(p.shape[1]):
        if (p.loc[index][i] < kth) & (p.loc[index][i] != 0):
            tmp.append(i+1)
    new_row.extend(tmp)
    workersToWrite.append(new_row)
pd.DataFrame(workersToWrite).to_csv('eclogs/workersToWrite.txt', index=False, header=False, sep = ' ')
workersToWrite = pd.read_csv('eclogs/workersToWrite.txt', header=None, sep = ' +', engine='python')
# (workersToWrite == 0).sum(axis=1)
workersToWrite<jupyter_output><empty_output><jupyter_text>生成读取策略<jupyter_code>circle_times = 5
read_list = list()

workersToWrite = workersToWrite.values.tolist()
p = pd.read_csv('eclogs/p.txt',header = None, sep = '\s+')

while(len(read_list) < circle_times*len(p)):
    rand = pd.DataFrame(np.random.rand(p.shape[0], p.shape[1]))
    read_matrix = (rand - p) < 0
    
    for index in read_matrix.index:
        new_row = []
        while(len(new_row) != k):
            rand = pd.DataFrame(np.random.rand(p.shape[0], p.shape[1]))
            read_matrix = (rand - p) < 0
            vc = read_matrix.loc[index].value_counts()
#             new_row = list(np.arange(workersToWrite[index].count(0)))
            new_row = [i for i,x in enumerate(workersToWrite[index]) if (x == 0)]
            if(len(new_row) > k):
                new_row = new_row[:k]
                
            tmp = []
            for j in range(0, p.shape[1]):
                if (read_matrix.loc[index][j] == True) and (j+1 in workersToWrite[index]):
                    tmp.append(workersToWrite[index].index(j+1))
            new_row.extend(tmp)
#             print(new_row)
        read_list.append(new_row)
pd.DataFrame(read_list).to_csv('eclogs/blocksToRead.txt', index=False, header=False, sep = ' ')<jupyter_output><empty_output><jupyter_text>生成随机或循环策略的读取概率<jupyter_code>file_num = 6
node_num = 8
k = 5
C = 8
cache_loc = np.random.rand(file_num, node_num) > 0.7142857
p_random = [[0]*node_num]*file_num
while(np.sum(cache_loc) != C):
    cache_loc = np.random.rand(file_num, node_num) > 0.7142857
    cnt = np.sum(cache_loc, axis=1)
    for i in range(len(cache_loc)):
        p_random[i] = [(k - cnt[i])/(node_num - cnt[i])]*node_num
        for j in range(len(cache_loc[i])):
            if cache_loc[i, j] == True:
                p_random[i][j] = 0
for line in p_random:
    for ele in line:
        print('%.4f'%ele, end=' ')
    print()
# cache_loc = np.random.rand(p.shape[0], p.shape[1]) > 0.7142857
# np.sum(cache_loc, axis=1)
# # p_random = [[0]*p.shape[1]]*p.shape[0]
# # [(k - cnt)/(p.shape[1]-cnt)]
len(p.loc[1])<jupyter_output><empty_output><jupyter_text>随机策略和sprout的写入策略<jupyter_code>import math

p = pd.read_csv('eclogs/p.txt',header = None, sep = '\s+')
workersToWrite = list()
# placement = determine_palcement()
cnt = np.sum(p == 0, axis=1)
for i in range(len(p)):

    new_row = [0] * int(cnt[i])
    
    tmp = []

    for j in range(len(p.loc[i])):
        if (p.loc[i][j] != 0):
            tmp.append(j+1)
    new_row.extend(tmp)
    
    workersToWrite.append(new_row)
    
for line in workersToWrite:
    for ele in line:
        print(str(ele), end=' ')
    print()
# pd.DataFrame(workersToWrite).to_csv('eclogs/workersToWrite.txt', index=False, header=False, sep = ' ')<jupyter_output>0 0 2 3 4 6 7 8 
0 0 1 2 4 5 6 7 
1 2 3 4 5 6 7 8 
0 0 1 3 4 5 6 8 
1 2 3 4 5 6 7 8 
0 0 2 3 4 5 7 8<jupyter_text>循环策略的写入策略<jupyter_code>import math

p = pd.read_csv('eclogs/p.txt',header = None, sep = '\s+')
workersToWrite = list()
# placement = determine_palcement()
cnt = np.sum(cache_loc, axis=1)
for i in range(len(p)):

    new_row = []

    for j in range(len(p.loc[i])):
        if (p.loc[i][j] == 0):
            new_row.append(0)
        else:
            new_row.append(j+1)
    workersToWrite.append(new_row)
    
for line in workersToWrite:
    for ele in line:
        print(str(ele), end=' ')
    print()<jupyter_output>1 2 3 4 5 6 0 8 
1 2 3 4 5 6 7 8 
1 2 3 4 5 6 7 8 
0 2 3 0 5 6 0 0 
1 0 3 0 5 6 7 8 
1 2 3 4 0 6 7 8
{'fraction_non_alphanumeric': 0.16643258426966293, 'fraction_numerical': 0.03300561797752809, 'mean_word_length': 2.7017543859649122, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '32438', 'license': 'Apache-2.0', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2322, 'n_words': 484, 'path': 'python/generate_policy.ipynb', 'repo_name': 'wushuhan/Alluxio-DataPlacement', 'size': '5696'}