<jupyter_start><jupyter_text>preprocess graph for contest<jupyter_code>fns = natsorted(
#     glob('contest-prep-0/*.json')
#     + glob('contest-prep-0/*.txt')
    glob('contest-0/*.json')
    + glob('contest-0/*.txt')
)
fns
for fn in fns:
    print(' in: ' + fn)
    with open(fn) as f:
        graph = json.load(f)
    graph['nodes'] = natsorted(graph['nodes'], key=lambda x:x['id'])
    
    nodes = [n['id'] for n in graph['nodes']]
    edges = [[e['source'], e['target']] for e in graph['edges']]
    
    G = nx.Graph()
    G.add_nodes_from(nodes)
    G.add_edges_from(edges)
    
    nodeCount = len(G.nodes)
    D, k2i = dict2tensor(dict(nx.all_pairs_shortest_path_length(G)))
    Adj,_ = dict2tensor(dict(G.adjacency()), fill=1)
    
    W = 1/(D**2+np.eye(nodeCount))
    js = graph2json(graph, D, W, graph['nodes'])
    
    fnout = fn.replace('-0', '-1').replace('txt', 'json')
    print(f'out: {fnout}')
    if not Path(Path(fnout).parent).exists():
        os.makedirs(Path(fnout).parent)
    with open(fnout, 'w') as f:
        json.dump(js, f, indent=2)
        print()
    print('-'*40)<jupyter_output>in: contest-0/automatic-1.txt
out: contest-1/automatic-1.json

----------------------------------------
 in: contest-0/automatic-2.txt
out: contest-1/automatic-2.json

----------------------------------------
 in: contest-0/automatic-3.txt
out: contest-1/automatic-3.json

----------------------------------------
 in: contest-0/automatic-4.txt
out: contest-1/automatic-4.json

----------------------------------------
 in: contest-0/automatic-5.txt
out: contest-1/automatic-5.json

----------------------------------------
 in: contest-0/automatic-6.txt
out: contest-1/automatic-6.json

----------------------------------------
 in: contest-0/automatic-7.txt
out: contest-1/automatic-7.json

----------------------------------------
 in: contest-0/automatic-8.txt
out: contest-1/automatic-8.json

----------------------------------------
 in: contest-0/automatic-9.txt
out: contest-1/automatic-9.json

----------------------------------------
 in: contest-0/automatic-10.txt<jupyter_text>html to be used in index<jupyter_code># for fn in fns:
#     graph_name = Path(fn).name.split('.')[0]
#     print(f"<option value='{graph_name}'>{graph_name}</option>")<jupyter_output><empty_output><jupyter_text>generate a graph<jupyter_code># fns = natsorted(
#     glob('_tsne_output/*.json')
#     +glob('_neato_sfdp_layouts0/*.dot'),
#     key=lambda x:x.split('/')[-1]
# )
# fns = [fn for fn in fns if 'grid' in fn and 'tnse']







# for fn in fns:
#     print(' in: ' + fn)
#     with open(fn) as f:
#         graph = json.load(f)
#     print(graph['nodes'])
#     for n in graph['nodes']:
#         n['id'] = str(n['id'])
        
#     for e in graph['edges']:
#         e['source'] = str(e['source'])
#         e['target'] = str(e['target'])
        
#     for p in graph['initPositions']:
#         p['id'] = str(p['id'])
    
#     for d, pos in zip(graph['nodes'], graph['initPositions']):
#         assert d['id'] == pos['id'], f"{d['id']} != {pos['id']}"
#         d['x'] = pos['x']
#         d['y'] = pos['y']
    
#     graph['nodes'] = natsorted(graph['nodes'], key=lambda x:x['id'])
    
#     for n in graph['nodes']:
#         print(n)
#     nodes = [n['id'] for n in graph['nodes']]
    
#     print(nodes)
#     edges = [[e['source'], e['target']] for e in graph['edges']]
    
#     G = nx.Graph()
#     G.add_nodes_from(nodes)
#     G.add_edges_from(edges)
    
#     nodeCount = len(G.nodes)
#     D, k2i = dict2tensor(dict(nx.all_pairs_shortest_path_length(G)))
#     Adj,_ = dict2tensor(dict(G.adjacency()), fill=1)
    
#     eye = torch.eye(nodeCount, device=device)
#     W = 1/(D**2+eye)
#     print(len(G.nodes), len(graph['nodes']))
#     js = graph2json(G, D, W, graph['nodes'])
#     fnout = fn.split('/')[-1].replace('.dot', '.json')
#     if 'neato' in fnout:
#         fnout = 'neato_layouts/' + fnout
#     elif 'sfdp' in fnout:
#         fnout = 'sfdp_layouts/' + fnout
#     elif 'tsne' in fnout:
#         fnout = 'tsne_layouts/' + fnout
#     else:
#         raise Exception
        
#     with open(fnout, 'w') as f:
#         json.dump(js, f, indent=2)
#         print(f'out: {fnout}')
#         print()
        
        
#     if 'neato.dot' in fn:
#         init_neato = [[d['x'], d['y']] for d in graph['nodes']]
#     if 'sfdp.dot' in fn:
#         init_sfdp = [[d['x'], d['y']] for d in graph['nodes']]
# #     if 'tsne.json' in fn:
# #         init_tsne = [[d['x'], d['y']] for d in graph['nodes']]
        
#         js['initPosition_neato'] = init_neato
#         js['initPosition_sfdp'] = init_sfdp
# #         js['initPosition_tsne'] = init_tsne
        
#         fnout = '_'.join(fnout.split('/')[-1].split('_')[:-1]) + '.json'
#         with open(fnout, 'w') as f:
#             json.dump(js, f, indent=2)
#             print('out: ' + fnout)
#         print('-'*20)
        
# nx.draw_networkx(G, pos={str(i):pos for i,pos in enumerate(init_neato)})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2590612002376708, 'fraction_numerical': 0.0142602495543672, 'mean_word_length': 2.0792682926829267, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '176972', 'license': 'Apache-2.0', 'n_tokens_mistral': 1894, 'n_tokens_neox': 1847, 'n_words': 342, 'path': 'tfjs-demo/evaluate/data/preprocess.ipynb', 'repo_name': 'abureyanahmed/sgd2', 'size': '5049'}