<jupyter_start><jupyter_text>plot graphs in database<jupyter_code>for g in gs.graphs.values():
    g.plot()
from gspan import gSpan
min_support = 30
alt = False
gs = gSpan(
    database_file_name='/data/experiments/DocRED/DataViewer/results/docred_train.data',
    min_support=min_support,
    min_num_vertices=2,
    # max_num_vertices=3,
    min_num_edges=1,
#     max_ngraphs=5,
    is_undirected=False,
    verbose=False,
    visualize=False,
    where=True,
    alternative_support=alt
)

"""Run the gSpan algorithm."""
# gathered = gs.run(gather=True)
# print(len(gathered))
# print(list(pdfs.edge for pdfs in gathered[0]))
gs.run()
df1 = gs._report_df
df1.to_pickle(f'all_patterns_support_{min_support}{"_alt" if alt else ""}.pkl')


gathered = gs.run(gather=True)
print(list(sorted(set(pdfs.gid for pdfs in gathered[0]))))
from perspective import PerspectiveWidget, Table
import pandas as pd
df1 = pd.read_pickle(f'all_patterns_support_{min_support}{"_alt" if alt else ""}.pkl')
PerspectiveWidget(df1)
print(df1['description'][58])
import itertools

with open('/data/experiments/DocRED/DataViewer/results/docred_train.data', 'r', encoding='utf8') as graph:
    transactions = []
    for line in graph:
        line = line.strip()
        t, *rest = line.split()
        if t == 't':
            c = int(rest[-1])
#             if c >= 5:
#                 break
            transactions.append({'v':[], 'e':[]})
        elif t == 'v':
            transactions[c][t].append(rest[-1])
        else: # t == 'e':
            transactions[c][t].append([int(rest[0]), int(rest[1]), rest[2]])
    


# for t in transactions:
#     print()
#     print(t)
#     for i, e in enumerate(t['e']):
#         print(f'#{i}: {t["v"][e[0]]}_{e[0]} {t["v"][e[1]]}_{e[1]} {e[2]}')

#     print()


def get_edge_patterns(pattern):
    pattern_dict = {'v':[], 'e':[]}
    for line in pattern.split('\n'):
        line = line.strip()
        t, *rest = line.split()
        if t == 't':
            continue
        elif t == 'v':
            pattern_dict[t].append(rest[-1])
        else: # t == 'e':
            pattern_dict[t].append([int(rest[0]), int(rest[1]), rest[2]])
    return [(pattern_dict['v'][start], pattern_dict['v'][end], label) for start, end, label in pattern_dict['e']], pattern_dict


def get_candidates(graph, edge_pattern):
    vs = graph['v']
    es = graph['e']
    candidate_edges = []
    for i, (start, end, label) in enumerate(es):
        if (vs[start], vs[end], label) == edge_pattern:
            candidate_edges.append(i)
    return candidate_edges


def trace(graph2, pattern2, steps, ignore_permutations=False):
#     vs2 = pattern2['v']
    es2 = pattern2
    candidate_paths = itertools.product(*steps)
    candidate_paths = [p for p in candidate_paths if len(p) == len(set(p))]
    gvs = graph2['v']
    ges = graph2['e']
    out = []
#     print("Candidates:", candidate_paths)
    varsets = list()
    for cp2 in candidate_paths:
        # Check if it's valid. We trust the labels and the types.
        # This is basically a type of variable resolution, heh.
        
        variables = dict()
        okay = True
#         print(cp2)
#         print(es2)
        if len(cp2) == len(es2):
            for step, e in zip(cp2, es2):
#                 print(step, e)
                v0, v1, _ = e
                start, end, _ = ges[step]
                if v0 in variables:
#                     print(f"{variables[v0]}!={start} ? (start): {variables}")
                    if variables[v0] != start:
                        okay = False
                        break
                if v1 in variables:
#                     print(f"{variables[v1]}!={end} ? (end): {variables}")
                    if variables[v1] != end:
                        okay = False
                        break

                variables[v0] = start
                variables[v1] = end
            if not okay:
                continue
            varset = set(variables.values())
            # If we didn't assign the same value to different variables
            if len(varset) == len(variables):
                # If we haven't used exactly this set of values before.
                # That is, if we change the order of values and it's still the same pattern,
                # then it's just one instance of that pattern. Permutation invariance.
                if varset in varsets and ignore_permutations:
                    continue
                varsets.append(varset)
                out.append(cp2)
            
    return out


def debug_one(df, index, verbose=True):
    tot = 0
    pattern = df['description'][index]
    support = df['support'][index]
    edge_patterns, pattern_dict = get_edge_patterns(pattern)
    if verbose:
        print(pattern)
    for tr in transactions:
        possible_steps = [get_candidates(tr, ep) for ep in edge_patterns]
#         possible_steps = [[10], [3], [5], [9]]
        founds = trace(tr, pattern_dict['e'], possible_steps)
        if verbose:
            print(founds)
        tot += len(founds)
    if verbose:
        print(index, tot, support)


def debug_all(df):
    for i, (pattern, support) in enumerate(zip(df['description'], df['support'])):
        tot = 0
        edge_patterns, pattern_dict = get_edge_patterns(pattern)
        for tr in transactions:
            possible_steps = [get_candidates(tr, ep) for ep in edge_patterns]
            founds = trace(tr, pattern_dict['e'], possible_steps)
    #         print(founds)
            tot += len(founds)
        if tot != support:
            print(i, tot, support)
    
# debug_one(df1, 19)
# debug_all(df1)

# from multiprocessing.dummy import Pool, Lock
# def make_corrections(df_in):
#     counts = [0 for _ in range(len(df_in['support']))]
#     lock = Lock()
#     def count_(inp):
# #         print(inp)
#         i, pattern = inp
#         tot = 0
#         edge_patterns, pattern_dict = get_edge_patterns(pattern)
#         for tr in transactions:
#             possible_steps = [get_candidates(tr, ep) for ep in edge_patterns]
#             founds = trace(tr, pattern_dict['e'], possible_steps)
#     #         print(founds)
#             tot += len(founds)
#         lock.acquire()
#         print(i, tot)
#         counts[i] = tot
#         lock.release()
#     pool = Pool(23)
# #     print(list(zip(*enumerate(df_in['description']))))
#     pool.map(count_, enumerate(df_in['description']))
#     df_in["counts"] = counts
#     return df_in

# df2 = pd.read_pickle(f'all_patterns_support_{min_support}_wcounts.pkl')
import multiprocessing as mp
import numpy as np

def count_(inp):
#         print(inp)
    counts = np.zeros(len(df1['support']), dtype=int)
    j, tr = inp
    tot = 0
    for i, pattern in enumerate(df1['description']):
        edge_patterns, pattern_dict = get_edge_patterns(pattern)
        possible_steps = [get_candidates(tr, ep) for ep in edge_patterns]
        founds = trace(tr, pattern_dict['e'], possible_steps, ignore_permutations=True)
        counts[i] += len(founds)
    print(f'{j},', end='')
    return counts

def make_corrections_2(df_in):
    lock = Lock()
    p = mp.Pool(mp.cpu_count())
    res = p.map_async(count_, enumerate(transactions))
    p.close()
    p.join()
    print()
#     print(np.sum(res.get(), axis=0))
    df_in["counts"] = np.sum(res.get(), axis=0)
    return df_in
    
# df2 = pd.read_pickle(f'all_patterns_support_{min_support}_wcounts_2.pkl')


df2 = make_corrections_2(df1)
df2.to_pickle(f'all_patterns_support_{min_support}_wcounts_noperms_gids.pkl')
PerspectiveWidget(df2)







# Now we want to calculate the coverage the patterns have of the dev/eval set.
# Then after we want to do the same for predictions (which will need to be formatted...)

# Document coverage:
# Basically do the counting but mark triples which are included.

def trace_coverage(graph2, pattern2, steps, coverage, ignore_permutations=True):
#     vs2 = pattern2['v']
    es2 = pattern2
    candidate_paths = itertools.product(*steps)
    candidate_paths = [p for p in candidate_paths if len(p) == len(set(p))]
    gvs = graph2['v']
    ges = graph2['e']
    if coverage is None:
        coverage = np.zeros(len(ges), dtype=int)
#     print("Candidates:", candidate_paths)
    varsets = list()
    for cp2 in candidate_paths:
        # Check if it's valid. We trust the labels and the types.
        # This is basically a type of variable resolution, heh.
        
        variables = dict()
        okay = True
#         print(cp2)
#         print(es2)
        if len(cp2) == len(es2):
            for step, e in zip(cp2, es2):
#                 print(step, e)
                v0, v1, _ = e
                start, end, _ = ges[step]
                if v0 in variables:
#                     print(f"{variables[v0]}!={start} ? (start): {variables}")
                    if variables[v0] != start:
                        okay = False
                        break
                if v1 in variables:
#                     print(f"{variables[v1]}!={end} ? (end): {variables}")
                    if variables[v1] != end:
                        okay = False
                        break

                variables[v0] = start
                variables[v1] = end
            if not okay:
                continue
            varset = set(variables.values())
            # If we didn't assign the same value to different variables
            if len(varset) == len(variables):
                # If we haven't used exactly this set of values before.
                # That is, if we change the order of values and it's still the same pattern,
                # then it's just one instance of that pattern. Permutation invariance.
                if varset in varsets and ignore_permutations:
                    continue
                varsets.append(varset)
                coverage[list(cp2)] = 1
    return coverage


def map_coverage(inp):
#     print(inp)
#     counts = np.zeros(len(df2['support']), dtype=int)
    j, tr = inp
    tot = 0
    coverage = None
    for i, pattern in enumerate(df2['description']):
        edge_patterns, pattern_dict = get_edge_patterns(pattern)
        possible_steps = [get_candidates(tr, ep) for ep in edge_patterns]
        coverage = trace_coverage(tr, pattern_dict['e'], possible_steps, coverage)
        if sum(coverage) == len(coverage):
            break
#         counts[i] += len(founds)
#     print(f'{j},')
    return np.sum(coverage), len(coverage)
#     return counts


def find_coverage(trs):
    lock = Lock()
    p = mp.Pool(mp.cpu_count())
    res = p.map_async(map_coverage, enumerate(trs))
    p.close()
    p.join()
    covered, total = zip(*res.get())
    print(f'{sum(covered)}/{sum(total)}: {sum(covered)/sum(total)}%')
#     print(sum(covered), sum(total))
#     print(np.sum(res.get(), axis=0))
#     df_in["counts"] = np.sum(res.get(), axis=0)
#     return df_in

find_coverage(transactions)
# map_coverage((0, transactions[0]))
with open('/data/experiments/DocRED/DataViewer/results/docred_dev.data', 'r', encoding='utf8') as graph:
    transactions_dev = []
    for line in graph:
        line = line.strip()
        t, *rest = line.split()
        if t == 't':
            c = int(rest[-1])
#             if c >= 5:
#                 break
            transactions_dev.append({'v':[], 'e':[]})
        elif t == 'v':
            transactions_dev[c][t].append(rest[-1])
        else: # t == 'e':
            transactions_dev[c][t].append([int(rest[0]), int(rest[1]), rest[2]])
find_coverage(transactions_dev)
# When I come back from vacation.
# I need to make sure to redo a lot of this.<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13297325102880658, 'fraction_numerical': 0.010888203017832648, 'mean_word_length': 2.2748455923638407, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '614327', 'license': 'MIT', 'n_tokens_mistral': 3801, 'n_tokens_neox': 3725, 'n_words': 927, 'path': 'analysis.ipynb', 'repo_name': 'rcapshaw/gSpan', 'size': '11664'}