<commit_before>import zmq
from kilogram import ListPacker

index_map = {}
values = []
index = 0

for line in open('labels.txt'):
    index_map[line.strip()] = index
    index += 1

for j, line in enumerate(open('wikipedia_pagelinks.tsv')):
    label, value = line.strip().split('\t')
    values.append(zip(*ListPacker.unpack(value))[0])
    if not j % 10000:
        print j
    j += 1

def uri_map(item):
    res = []
    uri, i = item.split('|--|')
    i = int(i)
    direct_neighbors = set(values[i])
    for neighbor in direct_neighbors:
        neighbor_uris = values[index_map[neighbor]]
        count = len(direct_neighbors.intersection(neighbor_uris))
        if count > 0:
            res.append(uri+"|--|"+neighbor+'\t'+str(count))
    return '\n'.join(res)

context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind("ipc:///tmp/wikipedia_edges")

while True:
    #  Wait for next request from client
    uri = socket.recv().strip()
    socket.send(uri_map(uri))
<commit_msg>server fix<commit_after>import zmq
from kilogram import ListPacker

index_map = {}
values = []
index = 0

for line in open('labels.txt'):
    index_map[line.strip()] = index
    index += 1

for j, line in enumerate(open('wikipedia_pagelinks.tsv')):
    label, value = line.strip().split('\t')
    values.append(zip(*ListPacker.unpack(value))[0])
    if not j % 10000:
        print j
    j += 1

def uri_map(item):
    res = []
    uri, i = item.split('|--|')
    i = int(i)
    direct_neighbors = set(values[i])
    for neighbor in direct_neighbors:
        neighbor_uris = values[index_map[neighbor]]
        count = len(direct_neighbors.intersection(neighbor_uris))
        if count > 0:
            res.append(neighbor+','+str(count))

    return uri+'\t'+' '.join(res)

context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind("ipc:///tmp/wikipedia_edges")

while True:
    #  Wait for next request from client
    uri = socket.recv().strip()
    socket.send(uri_map(uri))

{'fraction_non_alphanumeric': 0.1540785498489426, 'fraction_numerical': 0.010070493454179255, 'mean_word_length': 3.183157894736842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4218836', 'n_tokens_mistral': 718, 'n_tokens_neox': 696, 'n_words': 152}