<jupyter_start><jupyter_text>Target Creation<jupyter_code>cat1_dict={'Abarth':0,
'Aixam':0,
'AlfaRomeo':0,
'Audi':1,
'Austin':0,
'Bentley':0,
'BMW':1,
'Buick':0,
'Cadillac':0,
'Chevrolet':0,
'Chrysler':0,
'Citroen':0,
'Dacia':0,
'Daewoo':0,
'Daihatsu':0,
'Daimler':0,
'Dodge':0,
'DSAUTOMOBILES':0,
'Fiat':0,
'Ford':1,
'GMC':0,
'GreatWall':0,
'Honda':0,
'Hyundai':0,
'Infiniti':0,
'Isuzu':0,
'Iveco':0,
'Jaguar':0,
'JBA':0,
'Jeep':0,
'Kia':-1,
'Lancia':0,
'LandRover':1,
'Lexus':0,
'Lincoln':0,
'LondonTaxisInternational':0,
'Tesla':0,
'Maserati':0,
'Mazda':0,
'Mercedes-Benz':1,
'MEV':0,
'MG':0,
'Microcar':0,
'MINI':0,
'Mitsubishi':1,
'Morris':0,
'Nissan':0,
'Oldsmobile':0,
'Opel':0,
'Panther':0,
'Perodua':0,
'Peugeot':1,
'Pontiac':0,
'Porsche':0,
'Proton':0,
'Reliant':1,
'Renault':1,
'Rolls-Royce':0,
'Rover':0,
'Saab':0,
'SEAT':0,
'SKODA':0,
'Smart':0,
'Ssangyong':0,
'Subaru':1,
'Suzuki':0,
'Talbot':0,
'Tiger':0,
'Toyota':0,
'Triumph':0,
'TVR':0,
'Vauxhall':1,
'Volkswagen':0,
'Volvo':-1,
'Westfield':0,
'AstonMartin':0}

def cat1_transform(key):
    return cat1_dict[key]

cat1_tf=df['car_make'].apply(cat1_transform)
cat1_dict={'Freelancer':0, 
           'Homeworking':0, 
           'Unemployed':0, 
           'Self-Employed':-1, 
           'Contractor':-1, 
           'Permanent Employee':0}

cat2_tf=df['pol_employment_type'].apply(cat1_transform)
cat1_dict={'M':-0.1, 
           'F':0}

cat3_tf=df['gender'].apply(cat1_transform)
cat1_dict= {'Agriculture, Forestry, Fishing':0,
'Mining':0,
'Construction':-1,
'Manufacturing':0,
'Transportation & Public Utilities':-1,
'Wholesale Trade':0,
'Retail Trade':0,
'Finance, Insurance, Real Estate':-1,
'Services':0,
'Public Administration':1
}

cat4_tf=df['pol_employment_industry'].apply(cat1_transform)
cat1_dict={'Monday':0,
'Tuesday':0,
'Wednesday':0,
'Thursday':0,
'Friday':-1,
'Saturday':-1,
'Sunday':-0.5}

cat5_tf=df['collision_day'].apply(cat1_transform)
cat1_dict={'January':-1,
'February':-1,
'March':1,
'April':0,
'May':0,
'June':0,
'July':0,
'August':-1,
'September':0,
'October':0,
'November':-1,
'December':-1}

cat6_tf=df['collision_month'].apply(cat1_transform)
cat1_dict={'Yes':1,
'No':0}

cat7_tf=df['other_named_driver'].apply(cat1_transform)
cat1_dict={'AM':0,
'PM':-1}

cat8_tf=df['collision_time'].apply(cat1_transform)
cat1_dict={'Sunny/Clear':0,
'Partially cloudy':0,
'Cloudy':-1,
'Overcast':0,
'Rain':-1,
'Drizzle':0,
'Snow':-1,
'Stormy':-1,
'Tornadoes':-1,
'Thundersnows':-1,
'Fog':-1,
'Hurricanes':-1,
'Sandstorms':-1}

cat9_tf=df['collision_weather'].apply(cat1_transform)
def scaler(df, col):
    
    mean=df[col].mean()
    std=df[col].std()
    df[col + '_scl']=(df[col]-mean)/std
scaler(df, 'pol_holder_age')
scaler(df, 'car_value_as_new')
scaler(df, 'car_year')
scaler(df, 'postcode_rating')
scaler(df, 'credit_rating')
scaler(df,'car_top_speed')
scaler(df, 'pol_holder_licence_length')
target_name='large_collision'
mu, sigma = 0, 20
s = np.random.normal(mu, sigma, nrow)

# random noise
df[target_name] = s

df[target_name] = df[target_name] + (df['pol_holder_age_scl']**2)*-1 + \
(df['postcode_rating_scl'])*-1 + \
(df['credit_rating_scl']**5)*-1 + \
(df['car_top_speed_scl'])*-1 + \
(df['pol_holder_licence_length_scl']**3) + \
cat1_tf + cat2_tf + cat3_tf + cat4_tf + cat5_tf + cat6_tf + \
cat8_tf + cat9_tf + \
(df['car_value_as_new_scl']*df['pol_holder_age_scl']) + \
(df['car_top_speed_scl']*df['pol_holder_age_scl']) + \
cat5_tf * cat6_tf + \
cat7_tf * df['pol_holder_age_scl'] + \
df['car_year_scl']*df['car_top_speed_scl'] + \
(df['car_year_scl']*df['pol_holder_age_scl']) # age x car year interaction
df[target_name] = np.where(df[target_name] <= np.percentile(df[target_name], 20), 1, 0)
cols=['car_year', 
'car_make',
'car_value_as_new',
'pol_holder_age',
'postcode_rating',
'credit_rating',
'other_named_driver',
'criminal_convictions',
'car_modified_indicator',
'car_top_speed', 
'car_colour',
'pol_holder_licence_length',
'gender',
'pol_employment_type',
'pol_employment_industry',
'collision_day',
'collision_month',
'collision_time',
'collision_weather',
'collision_description']
for i in cols:

    # values for x-graph
    exposure=df.groupby(i)['exposure'].sum()
    obs=df.groupby(i)[target_name].mean()
    diff_band=sorted(set(df[i]))

    # Bar plot
    plt.rcParams["figure.figsize"] = (10,10)
    width = 0.5
    plt.bar(diff_band, exposure, width, color='gold', label='Diff', edgecolor='k')
    plt.xticks(rotation=90)
    plt.ylim(0, max(exposure)*3)
    plt.ylabel('Exposure')
    plt.xlabel(f'{i}')
    plt.title(f'One Way Plot of {i}')

    # Line plot
    axes2 = plt.twinx()
    axes2.plot(diff_band, obs, color='fuchsia', marker="s", markeredgecolor='black', label='Actual')
    # axes2.plot(diff_band, prop, color='blue', marker="D", markeredgecolor='black', label=f'Model 2 ({model2})')
    # axes2.plot(diff_band, curr, color='g', marker="^", markeredgecolor='black', label=f'Model 1 ({model1})')
    axes2.set_ylabel(target_name)

    # legend and settings
    plt.legend(loc="upper left")

    plt.show()<jupyter_output><empty_output><jupyter_text>Partitioning<jupyter_code>chunk=round(df.shape[0]/3,0)

# train
train=df.loc[df['id']<=chunk]

# validation
valid=df.loc[(df['id']>chunk) & (df['id']<=chunk*2)]

# test
test=df.loc[(df['id']>chunk*2)]

print(train.shape)
print(valid.shape)
print(test.shape)
# add a random rows with differing levels to validation dataset
lower_limit=chunk
upper_limit=chunk*2

edit_range=round(((upper_limit-lower_limit)/8),0)

criteria1=((valid['id']>lower_limit) & (valid['id']<=lower_limit+edit_range))

sports_premium_cars=['Porsche','Bentley', 'AstonMartin', 'Maserati','Rolls-Royce']
premium_cars=['AlfaRomeo','Audi', 'Infiniti', 'Lancia','Tesla','BMW', 'Jaguar', 'LandRover','Lexus','Mercedes-Benz']

criteria2=valid['car_make'].isin(sports_premium_cars)
criteria3=valid['car_make']=='Mercedes-Benz'

valid['car_make']=np.where(criteria1&criteria2, 'Lamborghini', valid['car_make'])
valid['car_make']=np.where(criteria1&criteria3, 'Tesla', valid['car_make'])
# add a random rows with differing levels to test dataset
lower_limit=chunk*2
upper_limit=chunk*3

edit_range=round(((upper_limit-lower_limit)/10),0)

criteria1=((test['id']>lower_limit) & (test['id']<=lower_limit+edit_range))

sports_premium_cars=['Porsche','Bentley', 'AstonMartin', 'Maserati','Rolls-Royce']
premium_cars=['AlfaRomeo','Audi', 'Infiniti', 'Lancia','Tesla','BMW', 'Jaguar', 'LandRover','Lexus','Mercedes-Benz']

criteria2=test['car_make'].isin(sports_premium_cars)
criteria3=test['car_make']=='Mercedes-Benz'

test['car_make']=np.where(criteria1&criteria2, 'Lamborghini', test['car_make'])
test['car_make']=np.where(criteria1&criteria3, 'Tesla', test['car_make'])<jupyter_output>/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:15: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  from ipykernel import kernelapp as app
/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:16: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  app.launch_new_instance()<jupyter_text>Final Dataset<jupyter_code># subset cols
cols = [#'id',
 'pol_ref',
 'name',
 'address',
 'gender',
 'car_make',
 'car_value_as_new',
 'car_year',
 'pol_holder_age',
 'pol_employment_type',
 'pol_employment_industry',
 'pol_holder_licence_length',
 'car_top_speed',
 'postcode_rating',
 'credit_rating',
 'other_named_driver',
 'criminal_convictions',
 'car_modified_indicator',
 'car_colour',
 'collision_date',
 'collision_description',
#  'collision_day',
#  'collision_month',
#  'collision_weather',
#  'collision_time',
#  'exposure',
#  'pol_holder_age_scl',
#  'car_value_as_new_scl',
#  'car_year_scl',
#  'postcode_rating_scl',
#  'credit_rating_scl',
#  'car_top_speed_scl',
#  'pol_holder_licence_length_scl',
 'large_collision']
train[cols].to_csv('data/train.csv', index=False)
test[cols].to_csv('data/test.csv', index=False)
valid[cols].to_csv('data/valid.csv', index=False)

cols.remove('large_collision')
valid[cols].to_csv('data/results.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20172776091524633, 'fraction_numerical': 0.028484706981088022, 'mean_word_length': 7.5159045725646125, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '172721', 'license': 'CNRI-Python', 'n_tokens_mistral': 3802, 'n_tokens_neox': 3660, 'n_words': 494, 'path': 'ML_Dataset_Generator.ipynb', 'repo_name': 'benturner993/ml_dataset_creation', 'size': '8566'}