<jupyter_start><jupyter_text>Generting Synthetic Insuance DataIn this notebook will demonstrate methods for generating synthetic insurance data. This notebook is written to accompany the blog post, ...<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from faker import Faker
import scipy.stats as st
from sdv.tabular import GaussianCopula
from sklearn.preprocessing import StandardScaler


pd.set_option('display.max_columns', 50)<jupyter_output>Bad key "text.kerning_factor" on line 4 in
/Users/smro/opt/anaconda3/envs/ml/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test_patch.mplstyle.
You probably need to get an updated matplotlibrc file from
https://github.com/matplotlib/matplotlib/blob/v3.1.3/matplotlibrc.template
or from the matplotlib source distribution<jupyter_text>Load data<jupyter_code>nor = pd.read_csv("norauto.csv")
usa = pd.read_csv("Auto_Insurance_Claims_Sample.csv")<jupyter_output><empty_output><jupyter_text>Basic data exploration<jupyter_code>nor.head()
usa.head()<jupyter_output><empty_output><jupyter_text>Compare distribution of total claims.<jupyter_code>fig, ax = plt.subplots(1, 2, figsize=(12,8))

ax[0].hist(usa["Claim Amount"], log=True)
ax[0].set_xlabel("Claim Amount")
ax[0].set_ylabel("Frequency")
ax[0].set_title("American Data")


ax[1].hist(nor["ClaimAmount"], log=True)
ax[1].set_xlabel("Claim Amount")
ax[1].set_ylabel("Frequency")
ax[1].set_title("Norweigian Data")<jupyter_output><empty_output><jupyter_text>For clearer view of issue with American dataset plot number of claims > 0 vs customers with 0 claims.<jupyter_code>usa["claim"] = "no claims"
nor["claim"] = "no claims"

usa.loc[usa["Claim Amount"] > 0, "claim" ] = "claimed"
nor.loc[nor["ClaimAmount"] > 0, "claim" ] = "claimed"
usa_vc = usa.claim.value_counts().to_frame()
usa_vc = usa_vc.reset_index()
usa_vc.columns = ["claims", "frequency"]
usa_vc["dataset"] = "American"
usa_vc["pc_freq"] = usa_vc["frequency"] / usa_vc["frequency"].sum() * 100

nor_vc = nor.claim.value_counts().to_frame()
nor_vc = nor_vc.reset_index()
nor_vc.columns = ["claims", "frequency"]
nor_vc["dataset"] = "Norwegian"
nor_vc["pc_freq"] = nor_vc["frequency"] / nor_vc["frequency"].sum() * 100

all_vc = usa_vc.append(nor_vc)
all_vc = all_vc.rename(columns={"dataset": "Dataset"})

all_vc
g = sns.catplot(
    data=all_vc, kind="bar",
    x="claims", y="pc_freq", hue="Dataset",
    ci="sd", palette="dark", alpha=.6, height=6
)
plt.xlabel("Claims > 0")
plt.ylabel("Percentage Frequency (%)")
plt.savefig("insurance_claims_comparison.png")<jupyter_output><empty_output><jupyter_text>Expect typical insurance portfolio to feature majority of non-claiments for their last period term. The Norwegian dataset matches these assumptions more closely.  Generating independent synthetic categorical data based on past observations<jupyter_code>def generate_distribution(series):
        
    vc = series.value_counts().to_frame()
    vc = vc.reset_index()
    vc.columns = ["cat", "count"]
    vc["pc"] = vc["count"] / vc["count"].sum()    
    return vc


def generate_categorical_data(data, columns, length):
    
    # Initialise blank dataframe
    generated_data = pd.DataFrame()
    
    # Scan through list of columns and generate synthetic data
    for col in cat_columns:

        # Get distribution
        counts = generate_distribution(data[col])
        
        # Generate categorical data
        col_data = np.random.choice(a=counts.cat, size=length, p=counts.pc)
        generated_data[col] = col_data
    
    return generated_data

cat_columns = ['Coverage', 'Education', 
               'EmploymentStatus', 'Gender',  'Location Code',
               'Marital Status', 'Policy Type', 'Policy', 'Claim Reason',
               'Sales Channel', 'Vehicle Class', 'Vehicle Size']

cat_data = generate_categorical_data(usa, cat_columns, 1000)
cat_data.head()
def visulise_distributions(original, synthetic, columns, path=None):
    
    fig, ax = plt.subplots(len(columns), 2, figsize=(14,10))
    
    
    for i, col in enumerate(columns, start=0):
#         print(col)
        
        # Original data
        vc = usa[col].value_counts()
        ax[0][0].set_title("Original Distributions")
        ax[i][0].bar(x=vc.index, height=vc)
        
        # Synthetic data
        vc = cat_data[col].value_counts()
        ax[0][1].set_title("Synthetic Distributions")
        ax[i][1].bar(x=vc.index, height=vc)
        
    plt.tight_layout()
    if path is not None:
        plt.savefig(path)
    
cat_columns = ['Coverage', 'Education', 'EmploymentStatus', 'Gender']

visulise_distributions(usa, cat_data, cat_columns, path="cat_dist_compare.png")<jupyter_output><empty_output><jupyter_text>Novel categorical data with Faker<jupyter_code>fake = Faker("en_US")

print("Novel Name:", fake.name())
print("Novel job:", fake.job())
print("Novel Address:", fake.local_latlng(country_code='US'))
# Generate independent synthetic numerical data
def best_fit_distribution(data, bins=200):
    
    # Bin data and find x midpoint
    y, x = np.histogram(data, bins=bins, density=True)
    x = (x + np.roll(x, -1))[:-1] / 2.0

    distributions = [        
    st.alpha, st.expon, st.gamma, st.norm, st.uniform]
    
    results = []
    
    for distribution in distributions:

        # Fit distribution
        params = distribution.fit(data)

        # Generate pdf for fitted distribution and measure fit
        pdf = distribution.pdf(x, loc=params[-2], scale=params[-1], *params[:-2])            
        sse = np.sum(np.power(y - pdf, 2.0))

        # Record fit result
        result = {"distribution": distribution.name,
                  "sse": sse,
                  "params": params}
        results.append(result)
    
    # Convert results to dataframe
    result_df = pd.DataFrame(results).sort_values("sse").reset_index(drop=True)
    
    return result_df

def model_numerical_data(data, cols, bins=200):
    
    best_fit = pd.DataFrame()
    for col in cols:
        result = best_fit_distribution(data[col], bins=bins)
        result["column"] = col
        best_fit = best_fit.append(result.iloc[0])
    
    return best_fit.reset_index(drop=True)

# Model numerical features for data generation
cols = ['Claim Amount',  
        'Total Claim Amount', 
        'Income', 'Monthly Premium Auto']
usa_sample = usa[cols].iloc[:500]

distribution_fit = model_numerical_data(usa_sample, cols, bins=200)
distribution_fit<jupyter_output>/Users/smro/opt/anaconda3/envs/ml/lib/python3.7/site-packages/scipy/stats/_distn_infrastructure.py:2429: RuntimeWarning: invalid value encountered in double_scalars
  Lhat = muhat - Shat*mu<jupyter_text>Generate synthetic data with representitive distributions and correlations using SDV<jupyter_code># Slice sample of USA data
cols = ['Claim Amount',  
        'Total Claim Amount', 
        'Income', 'Monthly Premium Auto']
usa_sample = usa[cols].iloc[:500]

# Fit Guassian copula
model = GaussianCopula()
model.fit(usa_sample)

# Generate synthetic data
new_data = model.sample(1000)
new_data.head()
usa_sample.corr() - new_data.corr()
fig, ax = plt.subplots(2, 1, figsize=(10,10), sharey=True)

ax[0].title.set_text("Original")
sns.heatmap(usa_sample.corr(), ax=ax[0])
ax[1].title.set_text("Synthetic")

sns.heatmap(new_data.corr(), ax=ax[1])
plt.savefig("corr_compare.png")
def visulise_continuous_distributions(original, synthetic, columns):
    
    fig, ax = plt.subplots(len(columns), 2, figsize=(10,10))
    
    
    for i, col in enumerate(columns, start=0):
        
        all_data = np.concatenate((original[col], synthetic[col]), axis=None)
        max_val = np.max(all_data)
        min_val = np.min(all_data)
        
        # Original data
        ax[i][0].hist(x=original[col])
        ax[i][0].title.set_text(col + ": original")
        ax[i][0].set_xlim(left=min_val, right=max_val)
        
        # Synthetic data
        ax[i][1].hist(x=synthetic[col])    
#         ax[i][0].title(col + ": synthetic")
        ax[i][1].title.set_text(col + ": synthetic")
        ax[i][0].set_xlim(left=min_val, right=max_val)
    
    fig.tight_layout()
    plt.savefig("copula_dist_compare.png")
        
model.get_distributions()
model.get_parameters()<jupyter_output><empty_output><jupyter_text>Simple feature contraintsSome of the features should not have negative values. Will clip values with numpy and see how this effects correlations.<jupyter_code>for col in new_data:
    new_data[col] = np.clip(new_data[col], a_min=0, a_max=None)
fig, ax = plt.subplots(2, 1, figsize=(10,10), sharey=True)

ax[0].title.set_text("Original")
sns.heatmap(usa_sample.corr(), ax=ax[0])
ax[1].title.set_text("Synthetic")

sns.heatmap(new_data.corr(), ax=ax[1])
plt.savefig("corr_compare_constraint.png")
visulise_continuous_distributions(usa_sample, new_data, columns=new_data.columns)<jupyter_output><empty_output><jupyter_text>Generate synthetic target<jupyter_code>usa.head()
usa.columns
cols = ["Income", 'Monthly Premium Auto', 'Months Since Last Claim', 'Number of Open Complaints', 'Number of Policies']
scaler = StandardScaler()
scaled_cols = [x + "_scaled" for x in cols]
usa[scaled_cols] = scaler.fit_transform(usa[cols])
usa.head()
# Generate synthetic frequency

# Linear function
w1 = 1.5
w2 = 1.1
w3 = 0.2
w4 = 1
w5 = 1.6
s = 0.0001
c = -1

usa["x"] = (
    w1 * usa["Income_scaled"] + 
    w2 * usa['Monthly Premium Auto_scaled'] + 
    w3 * usa['Months Since Last Claim_scaled'] + 
    w4 * usa['Number of Open Complaints_scaled'] + 
    w5 * usa['Number of Policies_scaled']) * s + c

a = 0.0001
b = -3

# Log link function for x to synthetic mean
usa["mu"] = np.exp(a * usa["x"] + b)

# Synthetic frequency drawn from poisson distribution
usa["frequency"] = np.random.poisson(usa["mu"])
usa["frequency"].hist()<jupyter_output><empty_output><jupyter_text>Compare synthetic target distribution to Norweigian data<jupyter_code>nor.head()
usa_vc = usa["frequency"].value_counts().to_frame()
usa_vc = usa_vc.reset_index()
usa_vc.columns = ["claims", "frequency"]
usa_vc["dataset"] = "Synthetic"
usa_vc["pc_freq"] = usa_vc["frequency"] / usa_vc["frequency"].sum() * 100

nor_vc = nor.NbClaim.value_counts().to_frame()
nor_vc = nor_vc.reset_index()
nor_vc.columns = ["claims", "frequency"]
nor_vc["dataset"] = "Norwegian Observations"
nor_vc["pc_freq"] = nor_vc["frequency"] / nor_vc["frequency"].sum() * 100

all_vc = usa_vc.append(nor_vc)
all_vc = all_vc.rename(columns={"dataset": "Dataset"})

all_vc
g = sns.catplot(
    data=all_vc, kind="bar",
    x="claims", y="pc_freq", hue="Dataset",
    ci="sd", palette="dark", alpha=.6, height=6
)
plt.xlabel("Claims")
plt.ylabel("Percentage Frequency (%)")
plt.savefig("sythetic_target_compare.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14084637466432076, 'fraction_numerical': 0.014816186683952218, 'mean_word_length': 4.335968379446641, 'pattern_counts': {'":': 8, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '35758', 'license': 'Apache-2.0', 'n_tokens_mistral': 3899, 'n_tokens_neox': 3705, 'n_words': 776, 'path': 'synthetic_insurance_data/synthetic_insurance_data.ipynb', 'repo_name': 'datascience-gft/blog_posts', 'size': '10799'}