<jupyter_start><jupyter_text>Creating a Holdout Data Set for the Nairobi Ambulance Challenge Importing packages<jupyter_code>import pandas as pd
import holidays<jupyter_output><empty_output><jupyter_text>Reading data<jupyter_code>df = pd.read_csv('../Inputs/Train.csv', parse_dates=['datetime'])
print(df.shape)
df.head()<jupyter_output>(6318, 4)<jupyter_text>Feature engineering<jupyter_code>def create_accident_features(data):
    dict_months = {1: "Jan", 2: "Feb", 3: "Mar", 4: "Apr", 5: "May", 6: "Jun",
               7: "Jul", 8: "Aug", 9: "Sep", 10: "Oct", 11: "Nov", 12: "Dec"}
    
    data["date"] = data["datetime"].apply(lambda x: x.date())
    data["holiday"] = data["date"].apply(lambda x: 1 if (x in holidays.Kenya()) else 0)
    data["time"] = data["datetime"].apply(lambda x: x.time())
    data["day"] = data["datetime"].apply(lambda x: x.day)
    data["weekday"] = data["datetime"].apply(lambda x: x.weekday())
    data["weekend"] = data["weekday"].apply(lambda x: 1 if x>4 else 0)
    data["month"] = data["datetime"].apply(lambda x: dict_months.get(x.month))
    data["half_year"] = data["datetime"].apply(lambda x: 1 if x.month<7 else 2)
    data["year"] = data["datetime"].apply(lambda x: x.year)
    
    return data
df = create_accident_features(df)
df.head()<jupyter_output><empty_output><jupyter_text>Splitting off a holdout set<jupyter_code>def split_accident_data(data, strategy, test_size=0.3, random_state=42):
    '''
    Splits the data set into a train and a test set.
    strategy:
        random = splits off random indices, using test_size and random_state parameters
        year_2019 = splits the days of 2019 off into a test set
        percentage_month = splits off the last days of every month to the test set according to the test_size
        2nd_half_2018 = oversamples the months from July to December 2018 by about 33%
    '''

    if strategy == "random":
        data = data.sample(frac=1, random_state=random_state).reset_index().drop("index", axis=1)
        split_at = round(data.shape[0] * test_size)
        data_train = data.iloc[split_at:, :]
        data_test = data.iloc[:split_at, :]
    elif strategy == "year_2019":
        data_train = data[data["datetime"] < "2019-01-01"]
        data_test = data[data["datetime"] >= "2019-01-01"]
    elif strategy == "percentage_month":
        split_at = round(30 * (1-test_size))
        data_train = data[data["day"] <= split_at]
        data_test = data[data["day"] > split_at]
    elif strategy == "2nd_half_2018":
        train_samples = round(data.shape[0] * (1-test_size))
        test_samples = round(data.shape[0] * test_size)
        data_train = data.sample(n=train_samples, weights="half_year", random_state=random_state)
        data_test = data.sample(n=test_samples, weights="half_year", random_state=random_state)
        
    return data_train, data_test
# Split accident data
df_train, df_test = split_accident_data(data=df, strategy="2nd_half_2018", test_size=0.3)

print("Train:")
print(df_train.head(3)["datetime"])
print(df_train.tail(3)["datetime"])
print(df_train.shape)

print("Test:")
print(df_test.head(3)["datetime"])
print(df_test.tail(3)["datetime"])
print(df_test.shape)
print(df_train["half_year"].value_counts())
print(df_test["half_year"].value_counts())<jupyter_output>1    2719
2    1704
Name: half_year, dtype: int64
1    1069
2     826
Name: half_year, dtype: int64
{'fraction_non_alphanumeric': 0.15108759553204, 'fraction_numerical': 0.03497942386831276, 'mean_word_length': 4.401587301587302, 'pattern_counts': {'":': 4, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '465445', 'license': 'MIT', 'n_tokens_mistral': 1255, 'n_tokens_neox': 1165, 'n_words': 263, 'path': 'Notebooks/Holdout_Set.ipynb', 'repo_name': 'AndOleAnd/Capstone_N_A_P', 'size': '3402'}