<jupyter_start><jupyter_text>Create pkl dataThis notebook can be used to create the pickled version of the block data. The paths and file names might have to be adapted.<jupyter_code>_raw_file_path = "/share/data/school_choice/"
_block_sfha_file_path = "Census Blocks w SFHA Flag.xlsx"
_block_demographic_file_path = "Data/SF 2010 blks 022119 with field descriptions (1).xlsx"
_block_frl_file_path = "dssg/juan_FRL_data.xlsx"

# block_sfha_df = pd.read_excel(_raw_file_path + _block_sfha_file_path, sheet_name=None, engine="openpyxl")
# block_demo_df = pd.read_excel(_raw_file_path + _block_demographic_file_path, sheet_name=None, engine="openpyxl")
block_frl_tk12_df = pd.read_excel(_raw_file_path + _block_frl_file_path, sheet_name=None, engine="openpyxl")
_block_frl_file_path_tk5 = "/share/data/school_choice_equity/raw_data/4YR Avg Block Tiebreaker Revised TK5 7.20.21 (JAM).xlsx"
block_frl_tk5_df = pd.read_excel(_block_frl_file_path_tk5, sheet_name=None, engine="openpyxl")
new_raw_file_path = "/share/data/school_choice_equity/data/"
def dict2pickles(data_dict, fname):
    keys = [key for key in data_dict.keys()]
    print(keys[0])
    data_dict[keys[0]].to_pickle(new_raw_file_path + fname + '_fields.pkl')
    data_dict[keys[1]].to_pickle(new_raw_file_path + fname + '_data.pkl')
dict2pickles(block_sfha_df, 'block_sfha')
dict2pickles(block_demo_df, 'block_demo')
dict2pickles(block_frl_tk12_df, 'block_frl_tk12')
dict2pickles(block_frl_tk5_df, 'block_frl_tk5')
_block_sfha_file_path = "../school_choice_equity/data/
# testing the code
import pandas as pd
import sys
from time import time
sys.path.append('../')

from src.d01_data.block_data_api import BlockDataApi

obj = BlockDataApi()
e = time()
print("Importing sfha...", end="")
obj.load_data(sfha=True)
print(time()-e)
e = time()
print("Importing frl...", end="")
obj.load_data(frl=True)
print(time()-e)
e = time()
print("Importing demo...", end="")
obj.load_data()
print(time()-e)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1258211217786761, 'fraction_numerical': 0.019706922688226377, 'mean_word_length': 7.048780487804878, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '362817', 'license': 'RSA-MD', 'n_tokens_mistral': 808, 'n_tokens_neox': 787, 'n_words': 141, 'path': 'notebooks/20210718-jjl-create-pkl-data.ipynb', 'repo_name': '18rberry/Operationalizing-Equity-Tiebreaker-in-SF', 'size': '1979'}