<jupyter_start><jupyter_text>_Building Graphs: Input Edges_<jupyter_code>import glob, os, sys, yaml
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import pprint
import seaborn as sns
import trackml.dataset
import torch
from torch_geometric.data import Data
import itertools
# append parent dir
# sys.path.append('..')
from src import draw_event
from src import compose_event<jupyter_output><empty_output><jupyter_text>_(+) - Input Data_<jupyter_code># mu- data (old)
# input_dir = './data_sets/pandaml/data_3.0_7.0_GeV/'

# mu+mu- data (current)
input_dir = './data_sets/pandaml/data_0.1_1.5_GeV/'

# pbarp data (coming)
# input_dir = os.environ['HOME']+'/current/2_deepana/pandaml/data/'
# Find All Input Data Files (hits.csv, cells.csv, particles.csv, truth.csv)
all_files = os.listdir(input_dir)

# Extract File Prefixes (use e.g. xxx-hits.csv)
suffix = '-hits.csv'
file_prefixes = sorted(os.path.join(input_dir, f.replace(suffix, ''))
                       for f in all_files if f.endswith(suffix))
# file_prefixes[:10]
event_id = 1
event_prefix = file_prefixes[event_id]
# load an event
hits, tubes, particles, truth = trackml.dataset.load_event(event_prefix)

# memory usage
mem_bytes = (hits.memory_usage(index=True).sum() 
             + tubes.memory_usage(index=True).sum() 
             + particles.memory_usage(index=True).sum() 
             + truth.memory_usage(index=True).sum())

print('{} memory usage {:.2f} MB'.format(os.path.basename(event_prefix), mem_bytes / 2**20))
hits.head()
hits.layer_id.unique()
tubes.head()
particles.head()
truth.head()<jupyter_output><empty_output><jupyter_text>_(+) - Build Event_- functions from _event_utils.py_<jupyter_code>event = compose_event(event_prefix,skewed=False)
draw_event(event,figsize=(10, 10));
event.head()
event.layer.unique()<jupyter_output><empty_output><jupyter_text>_Build Graphs_  _(A) - True Edges (Layerwise)_**True Graph** is the ground truth for GNN. It is built from creating edges from _`hits`_ from the same particle but in adjacent layers. For this purpose one has _`true_edges, hits = get_layerwise_edges(event)`_ function in the _`event_util.py`_.<jupyter_code>from LightningModules.Processing.utils.event_utils import get_layerwise_edges
true_edges, hits = get_layerwise_edges(event)
hits.head()
# lets get unique pids with freq (~ hits).
sel_pids, sel_pids_fr = np.unique(hits.particle_id, return_counts=True)
print(sel_pids)<jupyter_output>[ 1.  2.  3.  4.  5.  6.  7.  8.  9. 10.]<jupyter_text>_(B) - Input Edges (Layerwise)_**Input Graph** is the training input to GNN. It is build from edges from hits from all particles but in adjacent layers.- use same `hits` from `get_layerwise_edges()`- make `get_input_graph()` function similar to `get_layerwise_edges()`- add to Data variable.<jupyter_code># layer_groups.size()
# layer_groups.groups
# layer_groups.first()
# layer_groups.last()
# layer_groups.ngroups
# layer_groups.groups.keys()
def select_segments(hits1, hits2, filtering=True):
    
    # TODO: Impelement filtering flag
    # Start with all possible pairs of hits
    keys = ['event_id', 'r', 'phi', 'isochrone', 'sector_id']
    hit_pairs = hits1[keys].reset_index().merge(hits2[keys].reset_index(), on='event_id', suffixes=('_1', '_2'))
    
    if filtering:
        dSector = (hit_pairs['sector_id_1'] - hit_pairs['sector_id_2'])
        sector_mask = ((dSector.abs() < 2) | (dSector.abs() == 5))
        segments = hit_pairs[['index_1', 'index_2']][sector_mask]
    else:
        segments = hit_pairs[['index_1', 'index_2']]
        
    return segments

def construct_graph(hits, layer_pairs, filtering=True):
    """Construct one graph (e.g. from one event)"""

    # Loop over layer pairs and construct segments
    layer_groups = hits.groupby('layer')
    segments = []
    for (layer1, layer2) in layer_pairs:
        
        # Find and join all hit pairs
        try:
            hits1 = layer_groups.get_group(layer1)
            hits2 = layer_groups.get_group(layer2)
        # If an event has no hits on a layer, we get a KeyError.
        # In that case we just skip to the next layer pair
        except KeyError as e:
            logging.info('skipping empty layer: %s' % e)
            continue
        
        # Construct the segments
        segments.append(select_segments(hits1, hits2, filtering))
    
    # Combine segments from all layer pairs
    # segments = pd.concat(segments)
    return segments
# get number of layers, without skewed layers its just 18
n_layers = hits.layer.unique().shape[0]
print("total number of layers (w/o skewed): {}".format(n_layers))
# lets get pairs to adjacent layers
layers = np.arange(n_layers)
layer_pairs = np.stack([layers[:-1], layers[1:]], axis=1)
layer_pairs
# returns a list of indices from layer pairs.
segments = construct_graph(hits, layer_pairs, filtering=True)
# Combine segments from all layer pairs
# segments = pd.concat(segments)
# segments.describe()
# let's see the first layer pair (0th element)
# with pd.option_context('display.max_rows', None, 'display.max_columns', None):  # more options can be specified also
#    print(segments[0][["index_1", "index_2"]])
# get the layer pari [0,1] from segments
edge_index = segments[0].to_numpy().T
edge_index.shape
edge_index[0]
edge_index[1]
# hits.query("layer==0")
# hits.query("layer==1")
edge_index.shape[1] == len(segments[0])<jupyter_output><empty_output><jupyter_text>_(+) - Plotting Input Edges_<jupyter_code># plotting input_edges
plt.close('all')
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot()

p_ids = np.unique(event.particle_id.values)
det = pd.read_csv("src/stt.csv")
skw = det.query('skewed==0')
nkw = det.query('skewed==1') # one may look for +ve/-ve polarity
    
# detector layout
plt.scatter(skw.x.values, skw.y.values, s=20, facecolors='none', edgecolors='lightgreen')
plt.scatter(nkw.x.values, nkw.y.values, s=20, facecolors='none', edgecolors='coral')

# particle tracks
for pid in sel_pids:
    idx = hits.particle_id == pid
    ax.scatter(hits[idx].x.values, hits[idx].y.values, label='particle_id: %d' %pid)
    
# input edges
for iedge in range(edge_index.shape[1]):
#for iedge in range(7):
    pt1 = hits.iloc[edge_index[0][iedge]]
    pt2 = hits.iloc[edge_index[1][iedge]]
    ax.plot([pt1.x, pt2.x], [pt1.y, pt2.y], color='k', alpha=0.3, lw=1.5)

ax.set_title('Event ID # %d' % event_id)
ax.set_xlabel('x [cm]')
ax.set_ylabel('y [cm]')
ax.set_xlim(-40, 40)
ax.set_ylim(-40, 40)
ax.grid(False)
# ax.legend(fontsize=10, loc='best')
fig.tight_layout()
fig.savefig("input_edges.png")<jupyter_output><empty_output><jupyter_text>_(+) Sector-wise Filtering_* build edges only in neighouring sectors _i.e._ `|sector_id_i - sector_id_j| < 2`<jupyter_code># lets take first layer_pair and corresponding hits
layer_pairs[0]
layer_groups = hits.groupby('layer')
hits1 = layer_groups.get_group(0)
hits2 = layer_groups.get_group(1)
keys = ['event_id', 'r', 'phi', 'isochrone', 'sector_id']
hit_pairs = hits1[keys].reset_index().merge(hits2[keys].reset_index(), on='event_id', suffixes=('_1', '_2'))
hit_pairs
sector_mask = ((hit_pairs['sector_id_1'] - hit_pairs['sector_id_2']).abs() < 2)
sector_mask
hit_pairs[['index_1', 'index_2']].head()
hit_pairs[['index_1', 'index_2']][sector_mask].head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14248633879781422, 'fraction_numerical': 0.01680327868852459, 'mean_word_length': 4.684006211180124, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '407692', 'license': 'MIT', 'n_tokens_mistral': 2714, 'n_tokens_neox': 2624, 'n_words': 608, 'path': 'eda/stt_eda3.ipynb', 'repo_name': 'a-akram/ctd2022', 'size': '7320'}