<jupyter_start><jupyter_text>load data<jupyter_code>from sklearn.datasets import load_iris

iris = load_iris()

iris.data[:10]
iris.target<jupyter_output><empty_output><jupyter_text>standard$$x^{\prime}=\frac{x-\overline{X}}{S}$$<jupyter_code>from sklearn.preprocessing import StandardScaler

StandardScaler().fit_transform(iris.data)[:10]<jupyter_output><empty_output><jupyter_text>min-max$$x^{\prime}=\frac{x-M i n}{M a x-M i n}$$<jupyter_code>from sklearn.preprocessing import MinMaxScaler

MinMaxScaler().fit_transform(iris.data)[:10]<jupyter_output><empty_output><jupyter_text>normalization<jupyter_code>from sklearn.preprocessing import Normalizer

Normalizer().fit_transform(iris.data)[:10]<jupyter_output><empty_output><jupyter_text>binarizer<jupyter_code>from sklearn.preprocessing import Binarizer

Binarizer(threshold=3).fit_transform(iris.data)[:10]<jupyter_output><empty_output><jupyter_text>one hot encoder<jupyter_code>from sklearn.preprocessing import OneHotEncoder

OneHotEncoder().fit_transform(iris.target.reshape((-1, 1)))<jupyter_output>/Users/cuishunhua/miniconda3/envs/python36/lib/python3.6/site-packages/sklearn/preprocessing/_encoders.py:363: FutureWarning: The handling of integer data will change in version 0.22. Currently, the categories are determined based on the range [0, max(values)], while in the future they will be determined based on the unique values.
If you want the future behaviour and silence this warning, you can specify "categories='auto'".
In case you used a LabelEncoder before this OneHotEncoder to convert the categories to integers, then you can now use the OneHotEncoder directly.
  warnings.warn(msg, FutureWarning)<jupyter_text>missing value<jupyter_code>from numpy import vstack, array, nan
from sklearn.preprocessing import Imputer

Imputer().fit_transform(vstack((array([nan, nan, nan, nan]), iris.data)))[0]<jupyter_output>/Users/cuishunhua/miniconda3/envs/python36/lib/python3.6/site-packages/sklearn/utils/deprecation.py:58: DeprecationWarning: Class Imputer is deprecated; Imputer was deprecated in version 0.20 and will be removed in 0.22. Import impute.SimpleImputer from sklearn instead.
  warnings.warn(msg, category=DeprecationWarning)<jupyter_text>polynomial features<jupyter_code>from sklearn.preprocessing import PolynomialFeatures

PolynomialFeatures().fit_transform(iris.data)[:5]<jupyter_output><empty_output><jupyter_text>function transformer<jupyter_code>from numpy import log1p
from sklearn.preprocessing import FunctionTransformer

FunctionTransformer(log1p).fit_transform(iris.data)[:10]<jupyter_output>/Users/cuishunhua/miniconda3/envs/python36/lib/python3.6/site-packages/sklearn/preprocessing/_function_transformer.py:98: FutureWarning: The default validate=True will be replaced by validate=False in 0.22.
  "validate=False in 0.22.", FutureWarning)
/Users/cuishunhua/miniconda3/envs/python36/lib/python3.6/site-packages/sklearn/preprocessing/_function_transformer.py:98: FutureWarning: The default validate=True will be replaced by validate=False in 0.22.
  "validate=False in 0.22.", FutureWarning)
{'fraction_non_alphanumeric': 0.10837118754055808, 'fraction_numerical': 0.0227125243348475, 'mean_word_length': 8.370820668693009, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '567627', 'license': 'MIT', 'n_tokens_mistral': 1051, 'n_tokens_neox': 1008, 'n_words': 196, 'path': 'notebook/data_preprocessing.ipynb', 'repo_name': 'guerbai/ok-human', 'size': '3082'}