<jupyter_start><jupyter_text>This is the notebook used to generate the dataset used on the FKLearn Tutorial.ipynb* The FKLearn Tutorial notebook was used to introduce FKLearn on Nubank's Data Science Meetup and the idea was to give an overall idea on how and why you should use FKLearn<jupyter_code>import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from scipy.stats import truncnorm

def get_truncated_normal(mean=0, sd=1, low=0, upp=10):
    return truncnorm(
        (low - mean) / sd, (upp - mean) / sd, loc=mean, scale=sd)
ids = range(0, 10000)
months = range(1, 24)
unique_entries = np.array(np.meshgrid(ids, months)).T.reshape(-1, 2)
unique_entries.shape
np.power(np.array([months]), 0.1) 

X = get_truncated_normal(5000, 2000, 300, 20000)
income_by_id = X.rvs(len(ids))
income_by_id = np.repeat(income_by_id, len(months))
income_wrong_entry = np.random.binomial(1, 1 - 0.05, unique_entries.shape[0]).astype(bool)
income_array = np.where(income_wrong_entry == True, income_by_id.reshape(1, -1), 9999999)
income_array.shape
plt.hist(income_by_id, bins = range(0, 20000, 500)) 
customer_creation_date = []
for m_id in np.random.choice(len(months) * 31, len(ids)):
    customer_creation_date.append(np.datetime64("2017-01-01") + np.timedelta64(int(m_id), 'D'))
customer_creation_date = np.repeat(np.array(customer_creation_date), len(months))
phone_branches = ["samsung", "motorola", "iphone", "lg"]
random_phone = np.random.choice(4, len(ids), p=[0.15, 0.3, 0.25, 0.3])
cellphone_branch = [phone_branches[i] for i in random_phone]
cellphone_branch = np.repeat(cellphone_branch, len(months))
phone_factor = [0.7, 0.3, 0.9, 0.45]
cellphone_factor = [phone_factor[i] for i in random_phone]
cellphone_factor = np.repeat(cellphone_factor, len(months))
cellphone_factor
bureau_missing = np.random.binomial(1, 1 - 0.1, unique_entries.shape[0]).astype(bool)
Y = get_truncated_normal(500, 250, 0, 1000)
bureau_score = Y.rvs(unique_entries.shape[0])
monthly_factor = np.tile(np.power(np.array(months), 0.2), len(ids))
bureau_score = np.where(bureau_missing == True, bureau_score, np.nan) / monthly_factor
bureau_score
plt.hist(bureau_score, bins = range(0, 1000, 25)) 
willingness_to_spend = np.repeat(np.random.normal(500, 200, len(ids)), len(months))
willingness_to_spend
plt.hist(willingness_to_spend, bins = range(-1000, 1500, 50)) 
noise_feature = np.random.normal(1000, 100, unique_entries.shape[0])
a = (willingness_to_spend)
a_norm = (a - a.min()) / (a.max() - a.min())
b = (income_array)
b_norm = (b - b.min()) / (b.max() - b.min())
c = cellphone_factor * willingness_to_spend
c_norm = (c - c.min()) / (c.max() - c.min())
d = (np.where(np.isnan(bureau_score), 300.0, bureau_score))
d_norm = (d - d.min()) / (d.max() - d.min())
e = np.random.normal(1, 0.3, unique_entries.shape[0])
W = get_truncated_normal(2000, 300, 0, 50000)
spend = (a_norm + b_norm + c_norm + d_norm + e) * W.rvs(unique_entries.shape[0])
spend
spend.shape
income_array.shape
initial_df = (pd.DataFrame(
    unique_entries, columns=["id", "month"]
).assign(
       income=income_array.T,
       created_at=customer_creation_date.T,
       phone_type=cellphone_branch.T,
       bureau_score=bureau_score.T,
       spend_desire=willingness_to_spend.T,
       random_noise=noise_feature.T,
       monthly_spend=spend.T,
       month_date=lambda df: df.month * 31 + np.datetime64("2017-01-01")
)
.loc[lambda df: df.month_date >= df.created_at])

plt.plot(sorted(initial_df.month.unique()), initial_df.groupby("month").agg({"bureau_score": "mean"}))
initial_df
with_target_df = (initial_df.groupby("id", as_index=False).apply(lambda df: df
                                .assign(
                                       avg_last_2_months_spend=lambda x: x.monthly_spend.rolling(2).mean(),
                                       target=lambda x: x.avg_last_2_months_spend.shift(-2)))
                 .reset_index(drop=True))
with_target_df
with_target_df.to_csv("fklearn-tutorial-input-dataset.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14622057001239158, 'fraction_numerical': 0.04460966542750929, 'mean_word_length': 4.536351165980796, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '95185', 'license': 'Apache-2.0', 'n_tokens_mistral': 1687, 'n_tokens_neox': 1499, 'n_words': 301, 'path': 'docs/source/examples/fklearn_overview_dataset_generation.ipynb', 'repo_name': 'gbakie/fklearn', 'size': '4035'}