<jupyter_start><jupyter_text>Explore the method dataset(C) <NAME>, 2020, Mindscan<jupyter_code>import sys
sys.path.insert(0,'../src')
from de.mindscan.fluentgenesis.bpe.bpe_model import BPEModel
from de.mindscan.fluentgenesis.dataprocessing.method_dataset import MethodDataset


model = BPEModel("16K-full", "../src/de/mindscan/fluentgenesis/bpe/")
model.load_hparams()
dataset_directory = model.get_data_source_path()


method_dataset = MethodDataset()
method_dataset.loadDataset(dataset_directory = dataset_directory + "_ml1_bl1_to_bl16")

# filter= (lambda method_data : ((method_data['length_encoded_method_name']==1)&(method_data['length_encoded_method_body']<=16) & (method_data['length_encoded_method_body']>0)))
# method_dataset.loadFilterSaveDataset(dataset_directory = dataset_directory, filter=filter, output_suffix="_ml1_bl1_to_bl16")
number_of_methods = 0;


method_name_length_histogram = {}
method_body_length = {}
method_names = {}

method_data = method_dataset.read_next_method_data()

while method_data is not None:
    number_of_methods = number_of_methods + 1
    
    if number_of_methods % (256*1024) is 0:
        print(str(number_of_methods / 1024) + "k methoden geladen.")

    # statistic body length
    len_body = method_data['length_encoded_method_body']
    
    if len_body not in method_body_length:
        method_body_length[len_body] = 1
    else:
        method_body_length[len_body] = method_body_length[len_body] + 1

    # Statistik method length
    len_enc_method_name = method_data['length_encoded_method_name']
    if len_enc_method_name not in method_name_length_histogram:
        method_name_length_histogram[len_enc_method_name] = 1
    else:
        method_name_length_histogram[len_enc_method_name] = method_name_length_histogram[len_enc_method_name] + 1
    
    if not method_data['class_name'] == method_data['method_name']:
        name = method_data['method_name']
        if name not in method_names:
            method_names[name] = 1
        else:
            method_names[name] = method_names[name] + 1

    method_data = method_dataset.read_next_method_data()        
        
    # länge methoden namen in token
    # länge methodenbody in token
    # aufsammeln aller methodennamen und deren occurence

method_dataset.finish()
print(number_of_methods)

for len in sorted(method_name_length_histogram.keys()):
    count = method_name_length_histogram[len]
    print("Method Name len " + str(len) + " #count: " + str(count))

all_body_count = 0
for len in sorted(method_body_length.keys()):
    count = method_body_length[len]
    all_body_count = all_body_count + count
    print("Body len " + str(len) + " #count: " + str(count))
    
for name in sorted(method_names.keys()):
    count = method_names[name]
    print("MethodName '"+name+"' is implemented "+str(count)+" times.")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1021261763680725, 'fraction_numerical': 0.013942140118508192, 'mean_word_length': 4.821501014198783, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '208747', 'license': 'MIT', 'n_tokens_mistral': 983, 'n_tokens_neox': 951, 'n_words': 170, 'path': 'ipynb/exploring_method_dataset_0x01.ipynb', 'repo_name': 'mindscan-de/FluentGenesis-Classifier', 'size': '2876'}