<jupyter_start><jupyter_text>wordcount<jupyter_code>try:
    w_tree_1 = pd.read_pickle('pickle/wordcount_tiny_datanode1.pkl')
except:
    wordcount = Parse('wordcount_tiny_datanode1.out')
    wordcount.build_tree()
    trees = [node for node in wordcount.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    w_tree_1 = se_tree.value_counts().to_frame()
    w_tree_1.to_pickle('pickle/wordcount_tiny_datanode1.pkl')
try:
    w_tree_2 = pd.read_pickle('pickle/wordcount_tiny_datanode2.pkl')
except:
    wordcount = Parse('wordcount_tiny_datanode2.out')
    wordcount.build_tree()
    trees = [node for node in wordcount.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    w_tree_2 = se_tree.value_counts().to_frame()
    w_tree_2.to_pickle('pickle/wordcount_tiny_datanode2.pkl')
w_tree_1<jupyter_output><empty_output><jupyter_text>sort<jupyter_code>try:
    s_tree_1 = pd.read_pickle('pickle/sort_tiny_datanode1.pkl')
except:
    sort = Parse('sort_tiny_datanode1.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    s_tree_1 = se_tree.value_counts().to_frame()
    s_tree_1.to_pickle('pickle/sort_tiny_datanode1.pkl')
    
try:
    s_tree_2 = pd.read_pickle('pickle/sort_tiny_datanode2.pkl')
except:
    sort = Parse('sort_tiny_datanode2.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    s_tree_2 = se_tree.value_counts().to_frame()
    s_tree_2.to_pickle('pickle/sort_tiny_datanode2.pkl')
s_tree_2<jupyter_output><empty_output><jupyter_text>terasort<jupyter_code>try:
    t_tree_1 = pd.read_pickle('pickle/terasort_tiny_datanode1.pkl')
except:
    sort = Parse('terasort_tiny_datanode1.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    t_tree_1 = se_tree.value_counts().to_frame()
    t_tree_1.to_pickle('pickle/terasort_tiny_datanode1.pkl')
    
try:
    t_tree_2 = pd.read_pickle('pickle/terasort_tiny_datanode2.pkl')
except:
    sort = Parse('terasort_tiny_datanode2.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    t_tree_2 = se_tree.value_counts().to_frame()
    t_tree_2.to_pickle('pickle/terasort_tiny_datanode2.pkl')
t_tree_2<jupyter_output><empty_output><jupyter_text>Kmeans<jupyter_code>try:
    k_tree_1 = pd.read_pickle('pickle/kmeans_tiny_datanode1.pkl')
except:
    sort = Parse('kmeans_tiny_datanode1.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    k_tree_1 = se_tree.value_counts().to_frame()
    k_tree_1.to_pickle('pickle/kmeans_tiny_datanode1.pkl')
    
try:
    k_tree_2 = pd.read_pickle('pickle/kmeans_tiny_datanode2.pkl')
except:
    sort = Parse('kmeans_tiny_datanode2.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    k_tree_2 = se_tree.value_counts().to_frame()
    k_tree_2.to_pickle('pickle/kmeans_tiny_datanode2.pkl')<jupyter_output>161it [00:06, 23.33it/s]
100%|██████████| 551184/551184 [00:02<00:00, 234564.24it/s]
100%|██████████| 274653/274653 [00:02<00:00, 136166.76it/s]
661it [00:36, 18.17it/s]
100%|██████████| 2753394/2753394 [00:09<00:00, 283036.34it/s]
100%|██████████| 1372258/1372258 [00:18<00:00, 73932.44it/s]<jupyter_text>PageRank<jupyter_code>try:
    p_tree_1 = pd.read_pickle('pickle/pagerank_tiny_datanode1.pkl')
except:
    sort = Parse('pagerank_tiny_datanode1.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    p_tree_1 = se_tree.value_counts().to_frame()
    p_tree_1.to_pickle('pickle/pagerank_tiny_datanode1.pkl')
    
try:
    p_tree_2 = pd.read_pickle('pickle/pagerank_tiny_datanode2.pkl')
except:
    sort = Parse('pagerank_tiny_datanode2.out')
    sort.build_tree()
    trees = [node for node in sort.trees]
    hashtree = hash_tree(trees)
    se_tree = pd.Series(hashtree)
    p_tree_2 = se_tree.value_counts().to_frame()
    p_tree_2.to_pickle('pickle/pagerank_tiny_datanode2.pkl')
p_tree_2
c = pd.concat([w_tree_1, s_tree_1, t_tree_1, k_tree_1, p_tree_1], axis=1, sort=True)
c.columns = ['wordcount','sort', 'terasort', 'kmeans', 'pagerank']
c.dropna(axis=0,how='any')
c = pd.concat([w_tree_2, s_tree_2, t_tree_2, k_tree_2, p_tree_2], axis=1, sort=True)
c.columns = ['wordcount','sort', 'terasort', 'kmeans', 'pagerank']
c.dropna(axis=0,how='any')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13991944032223871, 'fraction_numerical': 0.04812380750476998, 'mean_word_length': 4.9345911949685535, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '133714', 'license': 'Apache-2.0', 'n_tokens_mistral': 2184, 'n_tokens_neox': 2042, 'n_words': 249, 'path': 'parseDataNode.ipynb', 'repo_name': 'Ding-Flash/incubator-retired-htrace', 'size': '4717'}