<jupyter_start><jupyter_text>Conversion of Yoochoose Sessions into Graphs<jupyter_code>!wget -q --show-progress https://github.com/sparsh-ai/stanza/raw/S969796/datasets/yoochoose1_64/raw/train.txt
!wget -q --show-progress https://github.com/sparsh-ai/stanza/raw/S969796/datasets/yoochoose1_64/raw/test.txt
import pickle

train_data = pickle.load(open('train.txt', 'rb'))
type(train_data)
train_data[0][:10]
train_data[1][:10]
len(train_data[0])
us_lens = [len(upois) for upois in train_data[0]] # user-session lengths
us_lens[:10]
len_max = max(us_lens) # maximum session length
len_max
us_pois = [upois + [0] * (len_max - le) for upois, le in zip(train_data[0], us_lens)]
print(*us_pois[:10])
us_msks = [[1] * le + [0] * (len_max - le) for le in us_lens] # user session mask
print(*us_msks[:10])
inputs = np.asarray(us_pois)
mask = np.asarray(us_msks)
len_max = len_max
targets = np.asarray(train_data[1])
length = len(inputs)<jupyter_output><empty_output><jupyter_text>![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAlsAAAEUCAYAAAAcI/NVAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAKboSURBVHhe7J0PXFRV3v8/TzLVjAb+oaZRRBQpoRp9gF2woIJCKij/PIi6aI9i4CZr8hiti2uaxsbjSjwYSyWu5qY+<KEY>)<jupyter_code>i = 2
j = 11

_inputs, _mask, _targets = inputs[i:j], mask[i:j], targets[i:j]

items, n_node, A, alias_inputs = [], [], [], []

for u_input in _inputs:
    n_node.append(len(np.unique(u_input)))

print(*n_node)
max_n_node = np.max(n_node)

for u_input in _inputs:
    node = np.unique(u_input)
    items.append(node.tolist() + (max_n_node - len(node)) * [0])
    u_A = np.zeros((max_n_node, max_n_node))
    for i in np.arange(len(u_input) - 1):
        if u_input[i + 1] == 0:
            break
        u = np.where(node == u_input[i])[0][0]
        v = np.where(node == u_input[i + 1])[0][0]
        u_A[u][v] = 1
    u_sum_in = np.sum(u_A, 0)
    u_sum_in[np.where(u_sum_in == 0)] = 1
    u_A_in = np.divide(u_A, u_sum_in)
    u_sum_out = np.sum(u_A, 1)
    u_sum_out[np.where(u_sum_out == 0)] = 1
    u_A_out = np.divide(u_A.transpose(), u_sum_out)
    u_A = np.concatenate([u_A_in, u_A_out]).transpose()
    A.append(u_A)
    alias_inputs.append([np.where(node == i)[0][0] for i in u_input])
print(*alias_inputs)
items
np.array(A).shape
A<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1593573599652627, 'fraction_numerical': 0.03430308293530178, 'mean_word_length': 4.907692307692308, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '423016', 'license': 'Apache-2.0', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1059, 'n_words': 158, 'path': '_notebooks/2022-01-05-yoochoose-session-graph.ipynb', 'repo_name': 'recohut/notebook', 'size': '59210'}