<commit_before>import cPickle

# needed for modifying the CSV
import pandas as pd
import numpy as np


def pickle_puf(input_filename, output_filename, protocol=2):
    '''Given a name for input and output files, opens input (assumes it's CSV)
    file and pickles it using the protocol (can be optionally set to 0 or 1,
    see docs for cPickle) with the output file name.
    '''
    csv_dataframe = pd.read_csv(input_filename)
    # this code is specific to OSPC processing of the CSV
    narray_dict = {}
    for col_name in csv_dataframe.columns.values:
        new_col = col_name.upper() if col_name.isalpha() else col_name
        narray_dict[new_col] = np.array(csv_dataframe[col_name])

    with open(output_filename, 'w') as target:
        cPickle.dump(narray_dict, target, protocol)

def unpicle(f_name):
    '''Given a file name opens the file and uses cPickle to load it as a python
    object.
    '''
    with open(f_name) as y_file:
        return cPickle.load(y_file)
<commit_msg>added length measurement to pickled object for subsequent referencing and added picked file to dir<commit_after>import cPickle

# needed for modifying the CSV
import pandas as pd
import numpy as np


def pickle_puf(input_filename, output_filename, protocol=2):
    '''Given a name for input and output files, opens input (assumes it's CSV)
    file and pickles it using the protocol (can be optionally set to 0 or 1,
    see docs for cPickle) with the output file name.
    '''
    csv_dataframe = pd.read_csv(input_filename)
    # this code is specific to OSPC processing of the CSV
    narray_dict = {'PUF DIM': len(csv_dataframe)}
    # We save the length of the PUF table for processing in calc
    for col_name in csv_dataframe.columns.values:
        new_col = col_name.upper() if col_name.isalpha() else col_name
        narray_dict[new_col] = np.array(csv_dataframe[col_name])

    with open(output_filename, 'w') as target:
        cPickle.dump(narray_dict, target, protocol)

def unpicle(f_name):
    '''Given a file name opens the file and uses cPickle to load it as a python
    object.
    '''
    with open(f_name) as y_file:
        return cPickle.load(y_file)

{'fraction_non_alphanumeric': 0.07363092498849516, 'fraction_numerical': 0.0027611596870685687, 'mean_word_length': 3.5481171548117154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249690', 'n_tokens_mistral': 716, 'n_tokens_neox': 671, 'n_words': 257}