<jupyter_start><jupyter_text>Generating CFG Samples<jupyter_code>import os
import yaml
import glob
import numpy as np
import random
import tqdm

from typing import Tuple, List, Dict
from matplotlib import pyplot as plt

from utils import yaml_load, yaml_write, get_section

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Utilitary FunctionsThese utilitary functions are used to select CFGs and generate adjacency and features matrices to be used by GNNs.These functions use the database file to several cases, such as: check the number of nodes in the CFGs, name of the files from a certian application and otimization sequence, *etc*.. The database file is a dictionary that contains information about applications and their optimization sequences, like this:```python{     Information about application 1    application_1: {         Application 1 optimized with sequence 0        0: {            cfg_file: '/path/to/cfg/file'            number_cfg_nodes: 10            cdfg_file: '/path/to/cdfg/file'            number_cdfg_nodes: 20            exectime: 1.234        }         Application 1 optimized with sequence 1        1: {               cfg_file: '/path/to/cfg/file'            number_cfg_nodes: 15            cdfg_file: '/path/to/cdfg/file'            number_cdfg_nodes: 22            exectime: 0.987        }    }         Information about application 2    application_2: {         Application 2 optimized with sequence 0        0: {            ...        }        ...    }    ... }```See `building_metadata` notebook for more information about database file.<jupyter_code>def filter_cfg_files_below_order(metadata: dict, order: int) -> dict:
    """Filter CFG files from metadata database which adjacency matrix is below some order.
    
    Parameters:
        metadata (dict): Dictionary of metadata datasabe. See `building_metadata` notebook for explanation about metadata
        order (int): Order of adjacency matrix to filter
    
    Returns:
        The filtered metadata database dictionary  
    """
    # Lets filter CFG files in which number of nodes is less an order
    filtered_graph_files = {
        benchmark_name: {
            opt_seq: opt_values 
            for opt_seq, opt_values in values.items() if opt_values['number_cfg_nodes'] < order
        } for benchmark_name, values in metadata.items()
    }

    # Remove applications that has less than 2 graphs
    filtered_graph_files = {benchmark_name: values for benchmark_name, values in filtered_graph_files.items() if len(values) > 2}
    print(f"Number of graphs with adjacency matrix below order {order}: {sum([len(values) for _, values in filtered_graph_files.items()])}")
    # Return filtered metadata
    return filtered_graph_files

def sample_cfg_files(metadata: dict, num_samples: int, sample_condition: callable = None, weighted: bool = True, tries: int = 10) -> List[Tuple[str, str, float]]:
    """Sample CFGs from metadata dict which satisfy a sample condition. Each sample contains 2 CFG files of the same application with different optimization sequences. 

    Parameters:
        metadata (dict): Dictionary of metadata datasabe.
        num_samples (int): Number of samples to pick
        sample_condition (callable): A function determining if the two CFG files satisfy a condition. The function must receive 2 parameters and return a
                                boolean indicating if the sample is valid (True) or not (False). The parameters are 2 dictionaries with information about the 
                                CFGs. Each dictionary contains the following information:
                                * cfg_file (str): /path/to/cfg/file
                                * cdfg_file (str): /path/to/cdfg/file
                                * number_cdfg_nodes (int): Number of nodes in CDFG representation
                                * number_cfg_nodes (int): Number of nodes in CFG representation
                                * exectime: Execution time of the application
        weighted (bool): True if selection of samples must by weighted by application and False otherwise
        
    Returns:
        A list of tuples, with `num_samples` elements, where each tuple has the following elements:
        * Path to first CFG file
        * Path to second CFG file
        * Relative speedup. Execution time of application from first CFG over execution time of application from second CFG
        Returns None if the informed number of samples is not reached
    """
    samples = []
    apps = list(metadata.keys())
    
    if weighted:
        # Weighted selection based on how many sequeces is in each app
        total_app_seq = sum([len(opt_sequences) for _, opt_sequences in metadata.items()])
        weights = [len(opt_sequences)/total_app_seq for _, opt_sequences in metadata.items()]
    else:
        # Equal weights for all elements
        weights = [1/len(apps)] * len(apps)
    
    # Returns None if no sample that satisfy condition match
    # Return 
    def try_sample(tries: int = 10):
        """Inner function used to pick a sample (i.e., two graph files app:opt1, app:opt2) that satisfy a sample condition

        Params:
            tries (int): NUmber of tries to pick a sample that satisfy condition.

        Returns:
            A 3-element tuple with:
            * The name of the application
            * The 1st optimization sequence for that application
            * The 2nd optimization sequence for that application
            Returns None if could not pick any sample that matches the sample condition 
        """
        # Tries to pick two CFGs for same application for 'tries' times
        for i in range(0, tries):
            # Choose an application
            app = np.random.choice(apps, p=weights)
            # Choose 2 optimization sequences for that application
            opt_seq1, opt_seq2 = random.sample(list(metadata[app].keys()), 2)
            # Does it have a sampling condition?
            if sample_condition is not None:
                # Applies the condition
                if sample_condition(metadata[app][opt_seq1], metadata[app][opt_seq2]) == True:
                    return (app, opt_seq1, opt_seq2)
            # If no sampling condition is given, just return the value 
            else:
                return (app, opt_seq1, opt_seq2)
        # Return None if no sample was retrieved
        return None
        
    # Pick 'num_samples' samples
    for i in range(0, num_samples):
        # Try to pick a sample for 'tries' times
        sample = try_sample(tries=tries)
        # Raises exception if no samples could be picked
        if not sample:
            raise Exception("Could not generate number of samples with the valid sample condition")
        
        # Generate a tuple (CFG1, CFG2, RelativeSpeedup) and append to samples list
        app, opt_seq1, opt_seq2 = sample
        t = (
            metadata[app][opt_seq1]['cfg_file'], 
            metadata[app][opt_seq2]['cfg_file'],
            metadata[app][opt_seq1]['exectime']/metadata[app][opt_seq2]['exectime']            
        )
        samples.append(t)

    # Return samples
    return samples

def generate_graph_feature_matrices(network_shape: Tuple[int, int], features_shape: Tuple[int, int], nodes: dict, calls: dict, datas: dict, nodes_features: dict) -> Tuple[np.array, np.array]:
    """Generate the graph's adjacency matrix and the feature matrix from sparse data (dict).
    
    Parameters:
        network_shape (tuple):  Shape of the graph's adjacency matrix used as input to GNN. 
                                The matrix is filled with zeros if the graph has less nodes than network_shape
        features_shape (tuple): Shape of the feature matrix used as input to GNN
                                The matrix is filled with zeros if the graph has less nodes than features_shape
        nodes (dict): Dictionary with nodes, as adjacency lists
        nodes_features (dict): Dictionary describing the feature vector of each node
        
    Returns:
        A 2-element tuple:
        * The graph matrix
        * The feature matrix
    """
    
    graph = np.full(network_shape, False, dtype='bool')
    features = np.zeros(features_shape, dtype='float32')
    
    # The graph is represented as a list of adjacency. Let's transform it to an adjacency matrix
    # Node is the node id and node_list is the adjacency list
    for node, node_list in nodes.items():
        for n in node_list:
            graph[node][n] = True
            
    for node, node_list in calls.items():
         for n in node_list:
            graph[node][n] = True

    for node, node_list in datas.items():
         for n in node_list:
            graph[node][n] = True
            
    # Lets read the features from the graphs
    # Node is the node id and node list is the feature vector
    for node, node_list in nodes_features.items():
        features[node] = node_list

    return graph, features

def load_sample_files(sample_tuple: Tuple[str, str, float], network_shape: Tuple[int, int], features_shape: Tuple[int, int]) -> Tuple[np.array, np.array, np.array]:
    """Given a tuple with CFG filenames, load each CFG and generate their adjacency and feature matrices.
    
    Parameters:
        sample_tuple (tuple): A 3-element tuple, with:
                * Path to first CFG file
                * Path to second CFG file
                * Relative speedup. Execution time of application from first CFG over execution time of application from second CFG
        network_shape (tuple):  Shape of the graph's adjacency matrix used as input to GNN. 
        features_shape (tuple): Shape of the feature matrix used as input to GNN
    
    Returns:
        A 3-element tuple with:
        * A matrix with 2 CFGs
        * A matrix with 2 CFGs features
        * The speedup (1-element matrix)
    """
    
    def load_graph_from_file(filename: str) -> Tuple[dict, dict]:
        """Inner function used to load the graph from a file
        
        Parameters:
            filename (str): Name of the file with CFG
        
        Returns:
            A 2-element tuple with the graph, represented as adjacency list and the features
        """
        x = yaml_load(filename)
        return x['nodes'], x['calls'], x['datas'], x['nodes_features']
    
    # Load graph from first file and generate the graph and feature matrices
    graph1, calls1, datas1, features1 = load_graph_from_file(sample_tuple[0])
    graph1, features1 = generate_graph_feature_matrices(network_shape, features_shape, graph1, calls1, datas1, features1)
    # Load graph from second file and generate the graph and feature matrices
    graph2, calls2, datas2, features2 = load_graph_from_file(sample_tuple[1])
    graph2, features2 = generate_graph_feature_matrices(network_shape, features_shape, graph2, calls2, datas2, features2)
 
    # Generate a matrix with the 2 graph matrices
    graphs = np.array([graph1, graph2], dtype='bool')
    # Generate a matrix with the 2 feature matrices
    features = np.array([features1, features2], dtype='float32') 
    # Generate 1-element speedup array
    speedup_array = np.array([sample_tuple[2]], dtype='float32')
    
    return graphs, features, speedup_array

def generate_samples(samples: List[Tuple[str, str, float]], network_shape: Tuple[int, int], feature_shape: Tuple[int, int], desc: str = 'Samples generated') -> Tuple[np.array, np.array, np.array]:
    """Generate samples used as input to a GNN
    
    Parameters:
        samples (list): List of tuple, where each tuple has the following elements:
                * Path to first CFG file
                * Path to second CFG file
                * Relative speedup. Execution time of application from first CFG over execution time of application from second CFG
        network_shape (tuple):  Shape of the graph's adjacency matrix used as input to GNN. 
        features_shape (tuple): Shape of the feature matrix used as input to GNN
        desc (str): Optional description for TQDM bar
        
    Returns:
        A 3-element tuple with:
        * A matrix with all samples, each sample has 2 CFGs
        * A matrix with all samples, each sample has 2 CFGs features
        * A matrix with all samples, each sample has the speedup
    """
    sampled_graphs = np.zeros((len(samples), 2, network_shape[0], network_shape[1]), dtype='bool')
    sampled_features = np.zeros((len(samples), 2, feature_shape[0], feature_shape[1]), dtype='float32')
    sampled_speedups = np.zeros((len(samples), 1), dtype='float32')
    
    # Some counters
    equal_graphs = 0
    equal_graphs_features = 0
    
    # Load each sample and store in input_graphs, input_features and speedups
    for i in tqdm.tqdm(range(0, len(samples)), desc=desc):
        try:
            # This function return a list of tuples, where each tuple is composed by:
            # np.array with 2 graphs, np.array with 2 features, speedup
            graphs, features, speedup = load_sample_files(samples[i], network_shape, feature_shape)           
            sampled_graphs[i] = graphs
            sampled_features[i] = features
            sampled_speedups[i] = speedup

            # Check if the matrices of graphs and features are equal for the same sample
            if np.array_equal(graphs[0], graphs[1]):
                equal_graphs += 1
                if np.array_equal(features[0], features[1]):
                    equal_graphs_features += 1
        except Exception as e:
            print(f'{type(e).__name__}: {e}')
            continue

    # Print some statistics
    print(f"Number of samples loaded: {len(samples)}")
    print(f"Graphs shape: {sampled_graphs.shape}")
    print(f"Features shape: {sampled_features.shape}")
    print(f"Speedups (target) shape: {sampled_speedups.shape}")
    print(f"Number of samples with equal graphs: {equal_graphs}")
    print(f"Number of samples with equal graphs and equal features: {equal_graphs_features}")
    
    return sampled_graphs, sampled_features, sampled_speedups

def get_output_filenames(data_dir: str, num_samples: int, network_shape: Tuple[int, int], prefix='cfgs') -> Tuple[str, str]:
    """Generate the file paths to store the samples
    
    Parameters:
        data_dir (str): Root data directory to store information
        num_samples (int): Number of samples generated
        network_shape (tuple):  Shape of the graph's adjacency matrix used as input to GNN
        
    Returns:
        A 2-element tuple with:
        * The name of the file to store the samples
        * The name of the file to store samples information (which CFG files were selected)
    """
    output_data_file = f"{prefix}_{num_samples}samples_{network_shape[0]}x{network_shape[1]}"
    output_data_file = os.path.join(data_dir, output_data_file)
    
    selected_data_file = f"selected_{prefix}_{num_samples}samples_{network_shape[0]}x{network_shape[1]}.yaml"
    selected_data_file = os.path.join(data_dir, selected_data_file)
    return output_data_file, selected_data_file

def save_data_file(output_data_file: str, output_samples_file: str, samples: List[Tuple[str, str, float]], graphs: np.array, features: np.array, speedups: np.array):
    """Save samples to a .npz file
    
    Parameters:
        output_data_file (str): Path to store the samples to npz file (graphs, feaures and speedups)
        output_samples_file (str): Path to store information about samples (which files correspond to each sample)
        samples (list): Files used to generate the samples
        graphs (np.array): Matrix with all graphs
        features (np.array): Matrix with all features
        speedups (np.array): Matrix with all speedups
        
    Returns:
        None
    """
    np.savez_compressed(output_data_file, graphs=graphs, features=features, speedups=speedups)
    print(f"Data saved to {output_data_file}.npz")
    
    yaml_write(output_samples_file, samples)
    print(f"Samples information saved to {output_samples_file}")<jupyter_output><empty_output><jupyter_text>Common variables for all shapes<jupyter_code># Default root directory to store all informations 
data_dir = './data'
# Metadata database file
metadata_file = './data/ccpe-applications-information-generated.cfgcd.yaml'
# Load metadata database
metadata_info = yaml_load(metadata_file)
print(f"Metadata loaded from '{metadata_file}'")<jupyter_output>Metadata loaded from './data/ccpe-applications-information-generated.cfgcd.yaml'<jupyter_text>Generating representations for (150x150) and (300x300) shapes The samples generation is composed by 4 steps:1. Remove CFG files where the number of CFG nodes is higher than a order (speficied by network_graph_shape variable). 2. Sample CFG files. This will generate a list of tuples, where each tuple is composed by: path to CFG1, path to CFG2, speedup between CFG1/CFG2. The two CFGs are from the same application, but with different optimization sequences.3. Generate representations from sampled CFG files (adjacency matrices, feature matrices and speedup matrices)4. Store samples to .npz file.<jupyter_code># Defining some variables
network_graph_shape = (150, 150)    # Network input graph's shape
network_features_shape = (150, 67)  # Network input feature's shape
n_samples = 20000                   # Number of samples to generate. Each sample is composed by 2 graphs
output_data_file, output_samples_file = get_output_filenames(data_dir, n_samples, network_graph_shape, prefix='generated2.cdfgcs')  # Paths to store samples

# 1. Filtering CFGs that number of nodes is below 'network_graph_shape'
cfg_files = filter_cfg_files_below_order(metadata_info, network_graph_shape[0])

# Conditions to compose each class of samples (A, B, C and D)
classA_cond = lambda opt1, opt2: opt1['exectime']/opt2['exectime'] < 0.45
classB_cond = lambda opt1, opt2: opt1['exectime']/opt2['exectime'] <= 0.80 and opt1['exectime']/opt2['exectime'] >= 0.45
classC_cond = lambda opt1, opt2: opt1['exectime']/opt2['exectime'] <= 1.33 and opt1['exectime']/opt2['exectime'] > 0.80
classD_cond = lambda opt1, opt2: opt1['exectime']/opt2['exectime'] > 1.33
# 2. Sample CFG files based on conditions. 1/4 of CFG files for each condition. 
print("Class A samples...")
sample_files =  sample_cfg_files(cfg_files, n_samples//4, sample_condition=classA_cond, tries=1000)
print("Class B samples...")
sample_files += sample_cfg_files(cfg_files, n_samples//4, sample_condition=classB_cond, tries=1000)
print("Class C samples...")
sample_files += sample_cfg_files(cfg_files, n_samples//4, sample_condition=classC_cond, tries=1000)
print("Class D samples...")
sample_files += sample_cfg_files(cfg_files, n_samples//4, sample_condition=classD_cond, tries=1000)

# 3. Generate samples from files
graphs, features, speedups = generate_samples(sample_files, network_graph_shape, network_features_shape)

# 4. Save all samples to a .npz file
save_data_file(output_data_file, output_samples_file, sample_files, graphs, features, speedups)<jupyter_output>Number of graphs with adjacency matrix below order 150: 17090
Class A samples...
Class B samples...
Class C samples...
Class D samples...
{'fraction_non_alphanumeric': 0.07888437922780726, 'fraction_numerical': 0.012956603184514517, 'mean_word_length': 2.962680412371134, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '406381', 'license': 'MIT', 'n_tokens_mistral': 5454, 'n_tokens_neox': 5179, 'n_words': 1951, 'path': 'generate_cfgs-1.ipynb', 'repo_name': 'otavioon/Code-Performance-Predictor', 'size': '19218'}