<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
from google.colab import drive 
drive.mount('/content/gdrive')
df=pd.read_csv('gdrive/My Drive/Colab Notebooks/Fraud/Fraud/fraud_data.csv')
import datetime

START_DATE = '2017-12-01'
startdate = datetime.datetime.strptime(START_DATE, "%Y-%m-%d")
df["Date"] = df['TransactionDT'].apply(lambda x: (startdate + datetime.timedelta(seconds=x)))

df['_Weekdays'] = df['Date'].dt.dayofweek
df['_Hours'] = df['Date'].dt.hour
df['_Days'] = df['Date'].dt.day
df['_Month'] = df['Date'].dt.month
df.drop('TransactionDT', axis=1, inplace=True)
train=pd.read_csv(הקובץ אחרי כל השינויים)
train['_Month']=df['_Month']
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
df.head()
df['_Hours']

df.to_csv('gdrive/My Drive/fraud_dataset.csv',index=False,compression='gzip')
df=pd.read_csv('gdrive/My Drive/fraud_dataset.csv',compression='gzip')
df=df1
df.loc[df['id_30'].str.contains('Windows', na=False), 'id_30'] = 'Windows'
df.loc[df['id_30'].str.contains('iOS', na=False), 'id_30'] = 'iOS'
df.loc[df['id_30'].str.contains('Mac OS', na=False), 'id_30'] = 'Mac'
df.loc[df['id_30'].str.contains('Android', na=False), 'id_30'] = 'Android'
df.loc[df['id_31'].str.contains('chrome', na=False), 'id_31'] = 'Chrome'
df.loc[df['id_31'].str.contains('firefox', na=False), 'id_31'] = 'Firefox'
df.loc[df['id_31'].str.contains('safari', na=False), 'id_31'] = 'Safari'
df.loc[df['id_31'].str.contains('edge', na=False), 'id_31'] = 'Edge'
df.loc[df['id_31'].str.contains('ie', na=False), 'id_31'] = 'IE'
df.loc[df['id_31'].str.contains('samsung', na=False), 'id_31'] = 'Samsung'
df.loc[df['id_31'].str.contains('opera', na=False), 'id_31'] = 'Opera'
emails = {'gmail': 'google', 'att.net': 'att', 'twc.com': 'spectrum', 'scranton.edu': 'other', 'optonline.net': 'other',
          'hotmail.co.uk': 'microsoft', 'comcast.net': 'other', 'yahoo.com.mx': 'yahoo', 'yahoo.fr': 'yahoo',
          'yahoo.es': 'yahoo', 'charter.net': 'spectrum', 'live.com': 'microsoft', 'aim.com': 'aol', 'hotmail.de': 'microsoft',
          'centurylink.net': 'centurylink', 'gmail.com': 'google', 'me.com': 'apple', 'earthlink.net': 'other', 
          'gmx.de': 'other', 'web.de': 'other', 'cfl.rr.com': 'other', 'hotmail.com': 'microsoft', 'protonmail.com': 'other',
          'hotmail.fr': 'microsoft', 'windstream.net': 'other', 'outlook.es': 'microsoft', 'yahoo.co.jp': 'yahoo',
          'yahoo.de': 'yahoo', 'servicios-ta.com': 'other', 'netzero.net': 'other', 'suddenlink.net': 'other',
          'roadrunner.com': 'other', 'sc.rr.com': 'other', 'live.fr': 'microsoft', 'verizon.net': 'yahoo',
          'msn.com': 'microsoft', 'q.com': 'centurylink', 'prodigy.net.mx': 'att', 'frontier.com': 'yahoo',
          'anonymous.com': 'other', 'rocketmail.com': 'yahoo', 'sbcglobal.net': 'att', 'frontiernet.net': 'yahoo',
          'ymail.com': 'yahoo', 'outlook.com': 'microsoft', 'mail.com': 'other', 'bellsouth.net': 'other',
          'embarqmail.com': 'centurylink', 'cableone.net': 'other', 'hotmail.es': 'microsoft', 'mac.com': 'apple',
          'yahoo.co.uk': 'yahoo', 'netzero.com': 'other', 'yahoo.com': 'yahoo', 'live.com.mx': 'microsoft', 'ptd.net': 'other',
          'cox.net': 'other', 'aol.com': 'aol', 'juno.com': 'other', 'icloud.com': 'apple'}
us_emails = ['gmail', 'net', 'edu']

for c in ['P_emaildomain', 'R_emaildomain']:
    df[c + '_bin'] = df[c].map(emails)
   
    
    df[c + '_suffix'] = df[c].map(lambda x: str(x).split('.')[-1])
   
    
    df[c + '_suffix'] = df[c + '_suffix'].map(lambda x: x if str(x) not in us_emails else 'us')
   
df.loc[:,"R_emaildomain"].mode()
df['P_emaildomain'].mode()
import gc
def setDevice(df):
   
    
    df['device_name'] = df['DeviceInfo'].str.split('/', expand=True)[0]

    df.loc[df['device_name'].str.contains('SM', na=False), 'device_name'] = 'Samsung'
    df.loc[df['device_name'].str.contains('SAMSUNG', na=False), 'device_name'] = 'Samsung'
    df.loc[df['device_name'].str.contains('GT-', na=False), 'device_name'] = 'Samsung'
    df.loc[df['device_name'].str.contains('Moto G', na=False), 'device_name'] = 'Motorola'
    df.loc[df['device_name'].str.contains('Moto', na=False), 'device_name'] = 'Motorola'
    df.loc[df['device_name'].str.contains('MOTO ', na=False), 'device_name'] = 'Motorola'
    df.loc[df['device_name'].str.contains('LG-', na=False), 'device_name'] = 'LG'
    df.loc[df['device_name'].str.contains('rv:', na=False), 'device_name'] = 'RV'
    df.loc[df['device_name'].str.contains('Huawei', na=False), 'device_name'] = 'Huawei'
    df.loc[df['device_name'].str.contains('ALE-', na=False), 'device_name'] = 'Huawei'
    df.loc[df['device_name'].str.contains('-L', na=False), 'device_name'] = 'Huawei'
    df.loc[df['device_name'].str.contains('Blade', na=False), 'device_name'] = 'ZTE'
    df.loc[df['device_name'].str.contains('BLADE', na=False), 'device_name'] = 'ZTE'
    df.loc[df['device_name'].str.contains('Linux', na=False), 'device_name'] = 'Linux'
    df.loc[df['device_name'].str.contains('XT', na=False), 'device_name'] = 'Sony'
    df.loc[df['device_name'].str.contains('HTC', na=False), 'device_name'] = 'HTC'
    df.loc[df['device_name'].str.contains('ASUS', na=False), 'device_name'] = 'Asus'

   # df.loc[df.device_name.isin(df.device_name.value_counts()[df.device_name.value_counts() < 200].index), 'device_name'] = "Others"
    df['had_id'] = 1
    gc.collect()
    
    return df

df=setDevice(df)

df1["device_name"].isnull().sum()
print(df["device_name"].unique())
df3.loc[:,"device_name"].mode()
df['device_name'].mode()
df=df2
df.loc[df.id_31.isin(df.id_31.value_counts()[df.id_31.value_counts() < 200].index), 'id_31'] = "Others"
df['id_31'].mode()
df.loc[df.C2.isin(df.C2\
                              .value_counts()[df.C2.value_counts() <= 350 ]\
                              .index), 'C2'] = "Others"
df['C2'].mode()
df['C2'].unique()
df.loc[df.C1.isin(df.C1\
                              .value_counts()[df.C1.value_counts() <= 400 ]\
                              .index), 'C1'] = "Others"
df['C1'].mode()
df2=df
df=df2
df.loc[df.addr1.isin(df.addr1.value_counts()[df.addr1.value_counts() <=4000 ].index), 'addr1'] = "Others"

df.loc[df.addr2.isin(df.addr2.value_counts()[df.addr2.value_counts() <= 50 ].index), 'addr2'] = "Others"
print(df["addr1"].unique())
df['addr1'].mode()
df['addr2'].unique()
df.loc[df.card3.isin(df.card3.value_counts()[df.card3.value_counts() <300].index), 'card3'] = "Others"

df['card3'].mode()
df.loc[df.card5.isin(df.card5.value_counts()[df.card5.value_counts() < 500].index), 'card5'] = "Others"
df['card5'].mode()
df.to_csv('gdrive/My Drive/dataSET.csv',index=False,compression='gzip')
data=pd.read_csv('gdrive/My Drive/dataSET.csv',compression='gzip')
int_cols=[]
float_cols=data.select_dtypes(include=['floating']).columns
pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)
for col in float_cols:
  flo=df[col].dropna()
float_cols = data.select_dtypes(include=['float'])
float_cols = float_cols.fillna(float_cols.median().round()) # median imputation
col_should_be_int = float_cols.applymap(float.is_integer).all()
float_to_int_cols = col_should_be_int[col_should_be_int].index
float_to_int_cols
list(float_to_int_cols)

df.loc[:, float_to_int_cols] = float_cols[float_to_int_cols].astype(int)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.23622469499932966, 'fraction_numerical': 0.019841801850113958, 'mean_word_length': 5.129827444535744, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '131701', 'license': 'MIT', 'n_tokens_mistral': 3093, 'n_tokens_neox': 2944, 'n_words': 423, 'path': 'DataSet.ipynb', 'repo_name': 'dinbav/Fraud', 'size': '7459'}