<jupyter_start><jupyter_text>This notebook processes the raw data and generates train and test sets and saves them in pickle diles.<jupyter_code>import pickle
import yaml
import pandas as pd
import numpy as np
def get_dates_arr_from_df(df):
    """
    This function converts the dates columns from a dataframe to an array of numbers
    """
    dates_2016 = []
    for i in range(10808):
        try:
            date = pd.datetime.combine(df.iloc[i, -1], df.iloc[i, 2])
            if date.year == 2016:
                dates_2016.append(date.timestamp())
        except:
            continue
    dates_2016_arr = np.array(dates_2016)
    dates_2016_so = np.sort(dates_2016_arr)
    dates_2016_so_0 = dates_2016_so - dates_2016_so[0]
    dates_2016_so_0_uni = np.unique(dates_2016_so_0)
    dates_2016_days = dates_2016_so_0_uni / 3600 / 24
    
    return dates_2016_days
# read the raw data
df = pd.read_excel('./NYPD_Complaint_Data_Current_YTD.xlsx', engine='openpyxl')
df.head()<jupyter_output><empty_output><jupyter_text>Only larceny crimes are included in the dataset. Crimes in Brooklyn are used as training data and crimes in Queens for testing.<jupyter_code>sub_df = df[df['OFNS_DESC'] =='PETIT LARCENY']
train_df = sub_df[sub_df['BORO_NM'] == 'BROOKLYN']
test_df = sub_df[sub_df['BORO_NM'] == 'QUEENS']
# clean the date
train_df['clean_date'] = pd.to_datetime(train_df['CMPLNT_FR_DT'])
test_df['clean_date'] = pd.to_datetime(test_df['CMPLNT_FR_DT'])
train_data = get_dates_arr_from_df(train_df)
test_data = get_dates_arr_from_df(test_df)
# write the train nd test datasets into pickle files.

with open('./train_data.p', 'wb') as f:
    pickle.dump([train_data], f)
with open('./test_data.p', 'wb') as f:
    pickle.dump([test_data], f)
# prepare config file for the inference algorithm

config = {'adapt_grad_step_size': False,
'data': '../../data/crime_dataset/nypd/train_data.p',
'convergence_criteria': 1.0e-04,
'grad_step_size': 0.01,
'hyper_updates': 1,
'infer_hypers': True,
'infer_max_intensity': True,
'init_val_for_kernel_cov': 50,
'init_val_for_memory_decay': 1.5,
'init_val_for_kernel_s_amp': 5.,
'init_val_for_kernel_s_cov': 200.,
'intensity_bound': 3.,
'lambda_a_prior': 1.3,
'lambda_b_prior': 0.1,
'memory_decay': 10.0,
'min_num_iterations': 10,
'num_inducing_points': 300,
'num_integration_points': 3000,
'real_data': True,
'save_steps': True,
'start_from_given': True,
'time_bound': 366,
'use_history': True}

with open(f'./settings_for_nypd_train_data.yml', 'w') as f:
    yaml.dump(config, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12965139052095573, 'fraction_numerical': 0.04465334900117509, 'mean_word_length': 4.9672897196261685, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '378334', 'license': 'MIT', 'n_tokens_mistral': 1070, 'n_tokens_neox': 961, 'n_words': 210, 'path': 'data/crime_datasets/nypd/nypd_data.ipynb', 'repo_name': 'noashin/NHGPS', 'size': '2553'}