<jupyter_start><jupyter_text>Feature Engineering.- Adjustment using power transformation to increase variance.- Transformation to unsigned integer of 32 bits (`uint32`). Why `uint32`?- Possible reduction in processing consumption.- Reduced memory usage.- Unsigned integer: for mathematical reasons it is good to avoid values equal to zero (or numerically approximated).- 32 bits: for preserve was necessary to preserve enough numerical precision. In the case of `uint16` and `uint8`, there would be a big loss of precision. Author: <NAME> [\[e-mail\]](mailto:'Israel%20Oliveira%20')<jupyter_code>%load_ext watermark
import pandas as pd
from sklearn.preprocessing import PowerTransformer
import numpy as np
from sys import float_info
from tqdm import tqdm

# from glob import glob

# import matplotlib.pyplot as plt
# %matplotlib inline
# from matplotlib import rcParams
# from cycler import cycler

# rcParams['figure.figsize'] = 12, 8 # 18, 5
# rcParams['axes.spines.top'] = False
# rcParams['axes.spines.right'] = False
# rcParams['axes.grid'] = True
# rcParams['axes.prop_cycle'] = cycler(color=['#365977'])
# rcParams['lines.linewidth'] = 2.5

# import seaborn as sns
# sns.set_theme()

# pd.set_option("max_columns", None)
# pd.set_option("max_rows", None)
# pd.set_option('display.max_colwidth', None)

# from IPython.display import Markdown, display
# def md(arg):
#     display(Markdown(arg))

from pandas_profiling import ProfileReport
#report = ProfileReport(#DataFrame here#, minimal=True)
#report.to

# import pyarrow.parquet as pq
# #df = pq.ParquetDataset(path_to_folder_with_parquets, filesystem=None).read_pandas().to_pandas()

# import json
# def open_file_json(path,mode='r',var=None):
#     if mode == 'w':
#         with open(path,'w') as f:
#             json.dump(var, f)
#     if mode == 'r':
#         with open(path,'r') as f:
#             return json.load(f)

# import functools
# import operator
# def flat(a):
#     return functools.reduce(operator.iconcat, a, [])

# import json
# from glob import glob
# from typing import NewType


# DictsPathType = NewType("DictsPath", str)


# def open_file_json(path):
#     with open(path, "r") as f:
#         return json.load(f)

# class LoadDicts:
#     def __init__(self, dict_path: DictsPathType = "./data"):
#         Dicts_glob = glob(f"{dict_path}/*.json")
#         self.List = []
#         self.Dict = {}
#         for path_json in Dicts_glob:
#             name = path_json.split("/")[-1].replace(".json", "")
#             self.List.append(name)
#             self.Dict[name] = open_file_json(path_json)
#             setattr(self, name, self.Dict[name])

# Run this cell before close.
%watermark -d --iversion -b -r -g -m -v
!cat /proc/cpuinfo |grep 'model name'|head -n 1 |sed -e 's/model\ name/CPU/'
!free -h |cut -d'i' -f1  |grep -v total
df = pd.read_csv('/work/data/creditcard.csv')
df<jupyter_output><empty_output><jupyter_text>Transformation to increase the variance.<jupyter_code>pt = PowerTransformer(method='box-cox', standardize=True)

unique_rel = {}
norm_cols = [col for col in df.columns if 'V' in col]
for col in tqdm(norm_cols):
    vals = df[col].to_numpy()
    vals = vals +abs(min(vals))+0.1
    trans_vals = pt.fit_transform(vals.reshape(-1, 1))
    unique_rel[col] = pd.DataFrame(trans_vals).nunique()/df[col].nunique()
    df[col] = trans_vals
#report = ProfileReport(df.query('Class == 1'), minimal=True)
#report.to_notebook_iframe()
#df.to_csv('/work/data/creditcard_trans_float.csv', index=False)<jupyter_output><empty_output><jupyter_text>How much unique values keep:<jupyter_code>sum([x.loc[0] for x in unique_rel.values()])/len(norm_cols)<jupyter_output><empty_output><jupyter_text>Transformation float to `uint32`<jupyter_code>th = 0.9999
n_digits = range(3,16)
max_int = np.iinfo(np.uint32).max - 1
unique_rel = {}
for col in tqdm(norm_cols):
    vals_ori = df[col].to_numpy()
    unique_vals = df[col].nunique()
    vals_rc = vals_ori.copy()
    nunique_vals_rc = 0.0
    for n in n_digits:
        vals = vals_ori*10**n
        vals = (vals + abs(vals.min()) + 1).round(0)
        if vals.max() >= max_int:
            break
        vals_rc = vals.round(0).astype(np.uint32)
        nunique_vals_rc = len(set(vals_rc))/unique_vals
        if nunique_vals_rc >= th:
            break
    unique_rel[col] = nunique_vals_rc
    df[col] = vals_rc<jupyter_output>100%|██████████| 28/28 [00:10<00:00,  2.73it/s]<jupyter_text>How much unique values keep:<jupyter_code>sum([x for x in unique_rel.values()])/len(norm_cols)
#df.to_csv('/work/data/creditcard_trans_int.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14325903484094352, 'fraction_numerical': 0.016879463319627787, 'mean_word_length': 4.067982456140351, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '409394', 'license': 'MIT', 'n_tokens_mistral': 1719, 'n_tokens_neox': 1642, 'n_words': 382, 'path': 'Feat_Eng.ipynb', 'repo_name': 'ysraell/creditcardfraud', 'size': '4630'}