<jupyter_start><jupyter_text>1) To set up your own data cleaning pipeline<jupyter_code>#Load the datasets
def read_dataset(name): #when only one dataset is provided as input
    import pandas as pd
    if name == "gpsa":
        df = pd.read_csv('../datasets/googleplaystore.csv', sep=',', encoding ='ISO-8859-1')
    elif name == "gpsu":
        df = pd.read_csv('../datasets/googleplaystore_reviews.csv', sep=',',encoding = 'ISO-8859-1')  
    elif name == "titanic":
        df = pd.read_csv('../datasets/titanic/titanic_train.csv', sep=',', encoding ='ISO-8859-1')
    elif name == "house":
        df = pd.read_csv('../datasets/house/house_train.csv', sep=',', encoding ='ISO-8859-1')
    else: 
        raise ValueError('Invalid dataset name')               
    return df

read_dataset("house").head()<jupyter_output><empty_output><jupyter_text>- Loading your data<jupyter_code>import learn2clean.loading.reader as rd 
import learn2clean.normalization.normalizer as nl 
import pandas as pd

# executing profiling function for one dataset as input
rd.profile_summary(read_dataset('house'), plot=False)


read_dataset('house')['SalePrice'].head() # the target variable is numerical 


# no encoding of the target variable
d_not_enc = rd.Reader(sep=',',verbose=True, encoding=False) 

# when you have two datasets as inputs: train and test datasets
house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
house_not_encoded = d_not_enc.train_test_split(house, 'SalePrice')
house_not_encoded['train'].head()
house_not_encoded['test'].head()

# otherwise for only one dataset as input, 
# train_test_split function will split the input dataset into train/test dataset
house_train_only  = ["../datasets/house/house_train.csv"]
house_train_only_not_encoded = d_not_enc.train_test_split(house_train_only, 'SalePrice')
house_train_only_not_encoded['train'].head()<jupyter_output>Reading csv : house_train.csv ...
Reading data ...
CPU time: 0.19720816612243652 seconds
Profiling datasets
        Attribute     Type  Num. Missing Values  Num. Unique Values           Sknewness   Kurtosis
0              Id  float64                  0.0              1460.0                   0       -1.2
1      MSSubClass  float64                  0.0                15.0             1.40621    1.57067
2     LotFrontage  float64                259.0               111.0  2.1608659947055435    17.3753
3         LotArea  float64                  0.0              1073.0             12.1951    202.544
4     OverallQual  float64                  0.0                10.0            0.216721  0.0918565
5     OverallCond  float64                  0.0                 9.0            0.692355    1.09852
6       YearBuilt  float64                  0.0               112.0           -0.612831  -0.442155
7    YearRemodAdd  float64                  0.0                61.0           -0.503044     -1.272
[...]<jupyter_text>- Normalize your data<jupyter_code># >> Examples of normalization
# The choice for the normalizer : 'ZS', 'MM','DS' or 'Log10'
#    Available strategies=
#       - 'ZS' z-score normalization
#       - 'MM' MinMax scaling
#       - 'DS' decimal scaling
#       - 'Log10 log10 scaling

import learn2clean.normalization.normalizer as nl 

# MM normalization with exclude = None, all numeric variables will be normalized
n1= nl.Normalizer(house_not_encoded.copy(),strategy='MM',exclude=None, verbose =False)
print('"LotArea" original variable\n',house_not_encoded['train']['LotArea'].head())
print('"LotArea" normalized variable\n', n1.transform()['train']['LotArea'].head())

#n1.transform()['train']
n1.transform()['target'].head()

#ZS normalization
n3= nl.Normalizer(house_not_encoded.copy(),strategy='ZS',exclude=None, verbose = False)
print('ZS normalized variables\n',n3.transform()['train'].head())

#DS scaling
n4= nl.Normalizer(house_not_encoded.copy(),strategy='DS',exclude=None, verbose = False)
print('DS normalized variables\n',n4.transform()['train'].head())

#Log10 scaling
n5= nl.Normalizer(house_not_encoded.copy(),strategy='Log10',exclude=None, verbose = False)
#print('Log10 normalized variables\n',n5.transform()['train'].head())
print('Log10 normalized variables none excluded in train dataset\n',n5.transform()['train']['LotArea'].head())
print('Log10 normalized variables none excluded in test dataset\n',n5.transform()['test']['LotArea'].head())

#Log10 scaling  excluding target variable 'SalePrice'
n6= nl.Normalizer(house_not_encoded.copy(),strategy='Log10',exclude='LotArea',verbose = False)

print('Log10 normalized variables LotArea excluded in train dataset \n',n6.transform()['train']['LotArea'].head())
print('Log10 normalized variables LotArea excluded  in test dataset \n',n6.transform()['test']['LotArea'].head())

n7= nl.Normalizer(house_not_encoded.copy(),strategy='Log10',exclude='Fare', verbose = False)
print('Log10 normalized variables none excluded\n', n5.transform()['train']['YearBuilt'].head())
print("Log10 normalized variables 'YearBuilt' excluded\n")
n7.transform()['train']['YearBuilt'].head()
# here YearBuilt is not excluded from normalization!


# To prevent that and avoid normalization for more than one numeric or year/month/date variables, 
# please change the data type into 'object' data type right after loading 
# and before normalization and learn2clean pipeline

house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
house_not_encoded = d_not_enc.train_test_split(house, 'SalePrice')

# not exhaustively
house_not_encoded['train']['YrSold'] = house_not_encoded['train']['YrSold'].astype('object')
house_not_encoded['train']['YearBuilt'] = house_not_encoded['train']['YearBuilt'].astype('object')
house_not_encoded['train']['MoSold'] = house_not_encoded['train']['MoSold'].astype('object')
house_not_encoded['train']['MiscVal'] = house_not_encoded['train']['MiscVal'].astype('object')
house_not_encoded['train']['GarageYrBlt'] = house_not_encoded['train']['GarageYrBlt'].astype('object')
house_not_encoded['train']['YearRemodAdd'] = house_not_encoded['train']['YearRemodAdd'].astype('object')

normalized_house = nl.Normalizer(house_not_encoded.copy(),strategy='DS',exclude='SalePrice', verbose = False).transform()
normalized_house['train'].head()<jupyter_output>Reading csv : house_train.csv ...
Reading data ...
CPU time: 2.4944140911102295 seconds
Profiling datasets
        Attribute     Type  Num. Missing Values  Num. Unique Values           Sknewness   Kurtosis
0              Id  float64                  0.0              1460.0                   0       -1.2
1      MSSubClass  float64                  0.0                15.0             1.40621    1.57067
2     LotFrontage  float64                259.0               111.0  2.1608659947055435    17.3753
3         LotArea  float64                  0.0              1073.0             12.1951    202.544
4     OverallQual  float64                  0.0                10.0            0.216721  0.0918565
5     OverallCond  float64                  0.0                 9.0            0.692355    1.09852
6       YearBuilt  float64                  0.0               112.0           -0.612831  -0.442155
7    YearRemodAdd  float64                  0.0                61.0           -0.503044     -1.272
8[...]<jupyter_text>- Replace missing values<jupyter_code>#>> Examples for missing value imputation
# Available strategies:
#            - 'EM': only for numerical variables; imputation based on
#                expectation maximization
#            - 'MICE': only for numerical variables  missing at random (MAR);
#                Multivariate Imputation by Chained Equations
#            - 'KNN', only for numerical variables; k-nearest neighbor
#                imputation (k=4) which weights samples using the mean squared
#                difference on features for which two rows both have observed
#                data
#            - 'RAND', 'MF': both for numerical and categorical variables;
#                replace missing values by randomly selected value in the 
#                variable domain or by the most frequent value in the variable
#                domain respectively
#            - 'MEAN', 'MEDIAN': only for numerical variables; replace missing
#                values by mean or median of the numerical variable respectvely
#            - or 'DROP' remove the row with at least one missing value

import learn2clean.imputation.imputer as imp

# replace missing values by the most frequent ones in the training and testing datasets

print('Number of missing values',normalized_house['train'].isnull().sum().sum()) 
imp1 = imp.Imputer(normalized_house.copy(),strategy='MF', verbose=True).transform()
imp2 = imp.Imputer(normalized_house.copy(),strategy='RAND', verbose=True).transform()
imp3 = imp.Imputer(normalized_house.copy(),strategy='DROP', verbose=True).transform()
imp4 = imp.Imputer(normalized_house.copy(),strategy='MEAN', verbose=True).transform() 
imp5 = imp.Imputer(normalized_house.copy(),strategy='KNN', verbose=True).transform()
imp6 = imp.Imputer(normalized_house.copy(),strategy='MEDIAN', verbose=True).transform()
imp6 = imp.Imputer(normalized_house.copy(),strategy='EM', verbose=True).transform()
imp6 = imp.Imputer(normalized_house.copy(),strategy='MICE', verbose=True).transform()<jupyter_output>Number of missing values 6965
>>Imputation 
* For train dataset
Before imputation:
Total 6965 missing values in ['LotFrontage', 'MasVnrArea', 'Alley', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual', 'Electrical', 'Fence', 'FireplaceQu', 'GarageCond', 'GarageFinish', 'GarageQual', 'GarageType', 'GarageYrBlt', 'MasVnrType', 'MiscFeature', 'PoolQC']
- 267 numerical missing values in ['LotFrontage', 'MasVnrArea']
- 6698 non-numerical missing values in ['Alley', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual', 'Electrical', 'Fence', 'FireplaceQu', 'GarageCond', 'GarageFinish', 'GarageQual', 'GarageType', 'GarageYrBlt', 'MasVnrType', 'MiscFeature', 'PoolQC']
Most frequent value for  1stFlrSF is: 0.2222222222222222
Most frequent value for  2ndFlrSF is: 9.999999977795539e-08
Most frequent value for  3SsnPorch is: 9.999999977795539e-08
Most frequent value for  BedroomAbvGr is: 0.5555555555555555
Most frequent value for  BsmtFinSF1 is: 9.99999997779[...]<jupyter_text>- Detect outliers and remove them<jupyter_code># >> Examples for outlier detection and removal
# Available strategies =
#            'ZS': detects outliers using the robust Zscore as a function
#            of median and median absolute deviation (MAD)
#            'IQR': detects outliers using Q1 and Q3 +/- 1.5*InterQuartile Range
#            'LOF': detects outliers using Local Outlier Factor

                
import learn2clean.outlier_detection.outlier_detector as out

#to remove rows having 30% and more ZSB-based outling values among the numerical variables
out1=out.Outlier_detector(house_not_encoded.copy(), strategy='ZSB', threshold = .3, verbose=True)
out1.transform()

#to remove rows having at least one IQR-based outlying value using threshold '-1'
out2=out.Outlier_detector(house_not_encoded.copy(), strategy='IQR', threshold = -1, verbose=False)
out2.transform()

#to remove rows having 40% and more ZSB-based outling values among the numerical variables; 
# since LOF requires non missing values, rows with NaN are also removed
out3=out.Outlier_detector(house_not_encoded.copy(), strategy='LOF', threshold = .4, verbose=False)
out3.transform()<jupyter_output>>>Outlier detection and removal:
* For train dataset
90 outlying rows have been removed:
with indexes: [4, 20, 53, 58, 65, 75, 112, 114, 118, 125, 159, 165, 167, 178, 185, 197, 198, 231, 251, 278, 304, 320, 321, 330, 344, 349, 363, 378, 389, 417, 434, 440, 477, 480, 496, 515, 520, 523, 527, 529, 533, 583, 591, 614, 635, 642, 649, 654, 661, 664, 691, 718, 745, 747, 759, 769, 798, 803, 825, 854, 888, 898, 918, 1007, 1024, 1031, 1032, 1044, 1046, 1067, 1068, 1142, 1169, 1173, 1182, 1184, 1228, 1230, 1243, 1266, 1268, 1284, 1298, 1313, 1350, 1353, 1373, 1386, 1405, 1449]

Outliers:
      1stFlrSF  2ndFlrSF  3SsnPorch Alley  BedroomAbvGr BldgType BsmtCond  \
4       1145.0    1053.0        0.0   NaN           4.0     1Fam       TA   
20      1158.0    1218.0        0.0   NaN           4.0     1Fam       TA   
53      1842.0       0.0        0.0   NaN           0.0     1Fam       TA   
58      1426.0    1519.0        0.0   NaN           3.0     1Fam       TA   
65      1143.0    1330.0     [...]<jupyter_text>- Detect duplicates and remove them<jupyter_code># >> Examples for duplicate detection and removal
# House dataset has no duplicate anyway
# Available strategies =
#        'ED':  exact duplicate detection/removal or
#        'AD':  for aproximate duplicate records detection and removal
#        based on Jaccard similarity 


# import the Duplicate_detector class
import learn2clean.duplicate_detection.duplicate_detector as dup

#Remove exact duplicates with 'ED' strategy of the Duplicate_detector class

dup.Duplicate_detector(house_not_encoded.copy(), strategy='ED', verbose=False).transform()

#Remove approximate duplicates with thresholding Jaccard similarity 
# using 'AD'strategy of the Duplicate_detector class
dup.Duplicate_detector(house_not_encoded.copy(), strategy='AD', threshold = .5, verbose=False).transform()<jupyter_output>Reading csv : house_train.csv ...
Reading data ...
CPU time: 0.18453502655029297 seconds
Profiling datasets
        Attribute     Type  Num. Missing Values  Num. Unique Values           Sknewness   Kurtosis
0              Id  float64                  0.0              1460.0                   0       -1.2
1      MSSubClass  float64                  0.0                15.0             1.40621    1.57067
2     LotFrontage  float64                259.0               111.0  2.1608659947055435    17.3753
3         LotArea  float64                  0.0              1073.0             12.1951    202.544
4     OverallQual  float64                  0.0                10.0            0.216721  0.0918565
5     OverallCond  float64                  0.0                 9.0            0.692355    1.09852
6       YearBuilt  float64                  0.0               112.0           -0.612831  -0.442155
7    YearRemodAdd  float64                  0.0                61.0           -0.503044     -1.272
[...]<jupyter_text>- Detect inconsistencies<jupyter_code># >> Examples for inconsistency detection 
# Available consistency checking strategies :
#            - 'CC': checks whether the data satisfy the constraints
#                specified in a 'file_name'_constraint.tdda stored in 'save' directory
#            - 'PC': checks whether the data satisfy the patterns
#                specified in 'file_name'_patterns.txt stored in 'save' directory

# import the Consistency_checker class                
import learn2clean.consistency_checking.consistency_checker as cc
import learn2clean.loading.reader as rd 
d_not_enc = rd.Reader(sep=',',verbose=False, encoding=False) 
house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
house_not_encoded = d_not_enc.train_test_split(house, 'SalePrice')
            
# discover the constraints from the input (train) dataset and store them in a file entitled 'house_discovered'_constraint.tdda in the 'save' directory
#cc.constraint_discovery(read_dataset('house'), file_name='house_discovered')

# discover the patterns from the input (train) dataset and store them in a file entitled 'house_discovered'_patterns.txt in the 'save' directory
#cc.pattern_discovery(read_dataset('house'), file_name='house_discovered')

# detect pattern violations with respect to a given file of patterns entitled 'house_example'_constraint.tdda" stored in the 'save' directory
cc.Consistency_checker(house_not_encoded, strategy='CC', file_name='house_example',verbose=True).transform()

# detect pattern violations with respect to a given file of patterns entitled 'house_example1'_patterns.txt" stored in the 'save' directory
# with too strong patterns resulting in an empty dataframe fro the training set
cc.Consistency_checker(house_not_encoded, strategy='PC', file_name='house_example1', verbose=False).transform()

# detect pattern violations with respect to a given file of patterns entitled 'house_example2'_patterns.txt" stored in the 'save' directory
cc.Consistency_checker(house_not_encoded, strategy='PC', file_name='house_example2', verbose=False).transform()<jupyter_output>>>Consistency checking
* For train dataset
Constraints from the file: house_example_constraints.tdda
Constraints passing: 6

Constraints failing: 1

FIELDS:

HouseStyle: 1 failure  4 passes  type ✓  min_length ✓  max_length ✓  max_nulls ✓  rex ✗

YearBuilt: 0 failures  2 passes  type ✓  max_nulls ✓

SUMMARY:

Constraints passing: 6
Constraints failing: 1
        field  failures  passes  type min_length max_length  max_nulls    rex
0  HouseStyle         1       4  True       True       True       True  False
1   YearBuilt         0       2  True        NaN        NaN       True    NaN
Row index with constraint failure:

[5, 8, 9, 15, 21, 46, 51, 61, 69, 77, 88, 93, 104, 108, 110, 121, 125, 127, 142, 149, 155, 163, 165, 170, 177, 183, 184, 185, 187, 198, 202, 204, 218, 239, 242, 249, 263, 267, 275, 286, 292, 296, 304, 307, 312, 325, 328, 335, 345, 352, 354, 361, 371, 380, 383, 386, 390, 394, 406, 418, 431, 436, 437, 439, 442, 448, 449, 459, 488, 491, 497, 514, 522, 545, 546, 555, 557, 56[...]<jupyter_text>- Select features<jupyter_code># >> Examples for Feature selection
# Available strategies=
#           'MR': using a default threshold on the missing ratio per variable,
#            i.e., variables with 20% (by default) and more missing values
#            are removed
#            'LC': detects pairs of linearly correlated variables and remove one
#            'VAR': uses threshold on the variance
#            'Tree': uses decision tree classification as model for feature
#                selection given the target set for classification task
#                'SVC': uses linear SVC as model for feature selection given
#                 the target set for classification task
#            'WR': uses the selectKbest (k=10) and Chi2 for feature selection
#                given the target set for classification task
#            'L1': uses Lasso L1 for feature selection given the target set for
#                regression task
#            'IMP': uses Random Forest regression for feature selection given
#                the target set for regression task

                
import learn2clean.feature_selection.feature_selector as fs
import learn2clean.loading.reader as rd 
d_not_enc = rd.Reader(sep=',',verbose=False, encoding=False) 
house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
house_not_encoded = d_not_enc.train_test_split(house, 'SalePrice')
  
#Available strategies for feature selection 
#        'MR': using a default threshold on the missing ratio per variable, i.e., variables
#                with 20% (by default) and more missing values are removed
#        'LC': detects pairs of linearly correlated variables and remove one
#        'VAR': uses threshold on the variance
#        'Tree': uses decision tree classification as model for feature selection given the target set for classification task
#        'SVC': uses linear SVC as model for feature selection given the target set for classification task
#        'WR': uses the selectKbest (k=10) and Chi2 for feature selection given the target set for classification task
#        'L1': uses Lasso L1 for feature selection given the target set for regression task
#        'IMP': uses Random Forest regression for feature selection given the target set for regression task

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'MR', threshold=0.1, exclude=None, verbose=True).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'LC', threshold=0.2,  exclude=None, verbose=True).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'L1',  exclude= None, threshold=.3,verbose=True).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'IMP', exclude = 'SibSp',verbose=True, threshold=.4).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'Tree',  exclude='Pclass',verbose=True).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'WR', exclude= None, verbose=True).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'WR', exclude= 'Sex', verbose=True).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'SVC',  exclude=None).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'VAR',  exclude=None).transform()

fs.Feature_selector(dataset = house_not_encoded.copy(), strategy= 'VAR',  exclude='Cabin').transform()<jupyter_output>>>Feature selection 
Before feature selection:
79 features 
Apply MR feature selection with missing threshold= 0.1
               missing_fraction
PoolQC                 0.995205
MiscFeature            0.963014
Alley                  0.937671
Fence                  0.807534
FireplaceQu            0.472603
LotFrontage            0.177397
GarageCond             0.055479
GarageFinish           0.055479
GarageQual             0.055479
GarageType             0.055479
GarageYrBlt            0.055479
BsmtExposure           0.026027
BsmtFinType2           0.026027
BsmtQual               0.025342
BsmtFinType1           0.025342
BsmtCond               0.025342
MasVnrArea             0.005479
MasVnrType             0.005479
Electrical             0.000685
MiscVal                0.000000
1stFlrSF               0.000000
Neighborhood           0.000000
MSZoning               0.000000
MSSubClass             0.000000
LowQualFinSF           0.000000
LotShape               0.000000
LotConfig           [...]<jupyter_text>>> Classification<jupyter_code>import learn2clean.classification.classifier as cl
#output is accuracy of classification for k=10 cross-validation and execution time 
#plus a detailed classification report if verbose = True
import learn2clean.loading.reader as rd 
d_enc = rd.Reader(sep=',',verbose=False, encoding=True) 
house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
house_encoded = d_enc.train_test_split(house, 'SaleCondition')


Cl1 = cl.Classifier(dataset = house_encoded,target = 'SaleCondition',strategy = 'LDA', verbose = False).transform()
Cl2 = cl.Classifier(dataset = house_encoded,target = 'SaleCondition',strategy = 'MNB',verbose = False).transform()
Cl3 = cl.Classifier(dataset = house_encoded,target = 'SaleCondition',strategy = 'NB',verbose = False).transform()
Cl4 = cl.Classifier(dataset = house_encoded,target = 'SaleCondition',strategy = 'CART',verbose = True).transform()<jupyter_output>>>Classification task

Accuracy of LDA result for 10 cross-validation : 0.803705337450375

Classification done -- CPU time: 0.15099120140075684 seconds

>>Classification task
Accuracy of Multinomial Naive Bayes classification for 10 cross-validation : 0.214

Classification done -- CPU time: 8.736706018447876 seconds

>>Classification task
Accuracy of Naive Naive Bayes classification for 10 cross-validation : 0.09572121746801941

Classification done -- CPU time: 0.1269359588623047 seconds

>>Classification task
{'mean_fit_time': array([0.00854752, 0.01225059, 0.01615422, 0.01999979, 0.02105217]), 'std_fit_time': array([0.00031048, 0.00020734, 0.00027083, 0.00084551, 0.00045577]), 'mean_score_time': array([0.00046787, 0.00049503, 0.00055208, 0.0006681 , 0.0005549 ]), 'std_score_time': array([5.52533913e-05, 8.20290235e-05, 5.80722472e-05, 2.98350208e-04,
       5.64673857e-05]), 'param_max_depth': masked_array(data=[3, 5, 7, 9, 10],
             mask=[False, False, False, False, False],[...]<jupyter_text>>> Regression<jupyter_code>import learn2clean.regression.regressor as rg
# output is MSE and computation time, with regression summary if verbose = True

import learn2clean.loading.reader as rd 
d_not_enc = rd.Reader(sep=',',verbose=False, encoding=False) 
house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
house_not_encoded = d_not_enc.train_test_split(house, 'SalePrice')
  
    
rg1 = rg.Regressor(dataset = house_not_encoded,target = 'SalePrice',strategy= 'LASSO', verbose = True).transform()

rg3 = rg.Regressor(dataset = house_not_encoded,target = 'SalePrice',strategy= 'OLS',verbose = True).transform()

rg2 = rg.Regressor(dataset = house_not_encoded,target = 'SalePrice',strategy= 'MARS',verbose = True).transform()<jupyter_output>>>Regression task
MSE values of cross validation
[[8.23014880e+08 9.43324263e+08 7.85739907e+08 2.11508746e+09
  1.90954945e+09 1.23725599e+09 1.00010508e+09 1.17530792e+09
  4.78814172e+09 9.80573235e+08]
 [8.20323363e+08 9.40411065e+08 7.83080690e+08 2.11443141e+09
  1.90636612e+09 1.23488480e+09 9.98348335e+08 1.16859980e+09
  4.79071404e+09 9.78396852e+08]
 [8.17519024e+08 9.37751124e+08 7.80568693e+08 2.11448351e+09
  1.90299198e+09 1.23321462e+09 9.97060396e+08 1.16237424e+09
  4.79389124e+09 9.75593687e+08]
 [8.14892384e+08 9.35343382e+08 7.78257153e+08 2.11454302e+09
  1.89985067e+09 1.23168256e+09 9.95937646e+08 1.15652492e+09
  4.79720251e+09 9.72969444e+08]
 [8.12430596e+08 9.33162697e+08 7.76123935e+08 2.11459638e+09
  1.89692381e+09 1.23029063e+09 9.94974879e+08 1.15102240e+09
  4.80066142e+09 9.70476627e+08]
 [8.10120644e+08 9.31187363e+08 7.74152175e+08 2.11463494e+09
  1.89419490e+09 1.22903007e+09 9.94160916e+08 1.14584442e+09
  4.80425725e+09 9.68091644e+08]
 [8.0795[...]<jupyter_text>>> Clustering<jupyter_code>import learn2clean.clustering.clusterer as ct
# clustering is applied to one dataset (i.e., the training set if two datasets are given in the path)
# output is silhouette, best k, and computation time, plus the training dataset with cluster IDs

ct.Clusterer(dataset = house_not_encoded,strategy= 'KMEANS', verbose=True).transform()
ct.Clusterer(dataset = house_not_encoded,strategy='HCA', verbose = True).transform()
ct.Clusterer(dataset = house_not_encoded,strategy='HCA', metric= 'euclidean', verbose = True).transform()
ct.Clusterer(dataset = house_not_encoded,strategy='HCA', metric= 'cosine', verbose = True).transform()
ct.Clusterer(dataset = house_not_encoded,strategy='HCA', metric= 'cityblock', verbose = True).transform()<jupyter_output>>>Clustering task
Note: The clustering is applied on the training dataset only.
Best silhouette = 0.9738  for k= 2
Quality of clustering 0.9738
Labels distribution:
0    1120
1       1
Name: cluster_ID, dtype: int64
Clustering done -- CPU time: 1.1059057712554932 seconds

>>Clustering task
Note: The clustering is applied on the training dataset only.
Best silhouette = 0.9738  for k= 2
Quality of clustering 0.9738
Labels distribution:
0    1120
1       1
Name: cluster_ID, dtype: int64
Clustering done -- CPU time: 0.31883811950683594 seconds

>>Clustering task
Note: The clustering is applied on the training dataset only.
Best silhouette = 0.9738  for k= 2
Quality of clustering 0.9738
Labels distribution:
0    1120
1       1
Name: cluster_ID, dtype: int64
Clustering done -- CPU time: 0.31737494468688965 seconds

>>Clustering task
Note: The clustering is applied on the training dataset only.
Best silhouette = 0.9738  for k= 2
Quality of clustering 0.3559
Labels distribution:
1    1016
0  [...]<jupyter_text>- Create your own pipeline<jupyter_code># create your preprocessing pipeline for classification

import learn2clean.loading.reader as rd 
import learn2clean.normalization.normalizer as nl 
import learn2clean.feature_selection.feature_selector as fs
import learn2clean.duplicate_detection.duplicate_detector as dd
import learn2clean.outlier_detection.outlier_detector as od
import learn2clean.imputation.imputer as imp
import learn2clean.classification.classifier as cl

d_not_enc = rd.Reader(sep=',',verbose=False, encoding=False) 
house  = ["../datasets/house/house_train.csv", "../datasets/house/test.csv"]
dataset= d_not_enc.train_test_split(house, 'SaleCondition')
  
# not exhaustively
dataset['train']['YrSold'] = dataset['train']['YrSold'].astype('object')
dataset['train']['YearBuilt'] = dataset['train']['YearBuilt'].astype('object')
dataset['train']['MoSold'] = dataset['train']['MoSold'].astype('object')
dataset['train']['MiscVal'] = dataset['train']['MiscVal'].astype('object')
dataset['train']['GarageYrBlt'] = dataset['train']['GarageYrBlt'].astype('object')
dataset['train']['YearRemodAdd'] = dataset['train']['YearRemodAdd'].astype('object')

# eliminate variables with more than30% missing values
d1 = fs.Feature_selector(dataset=dataset.copy(),strategy= 'MR', threshold=.3).transform()
# replace numeric null values by mean 
d2 = imp.Imputer(d1.copy(), strategy = 'MEAN',verbose=False).transform()
# replace remaining non numerical values by the most frequent
d2 = imp.Imputer(d2.copy(), strategy = 'MF',verbose=False).transform()
# eliminate 40 LOF outliers
d4 = od.Outlier_detector(d2, strategy='LOF', threshold= 0.4,verbose=False).transform()
# classify with LDA
cl.Classifier(d4,strategy = 'LDA', target = 'SaleCondition', verbose =True).transform()<jupyter_output>>>Feature selection 
Before feature selection:
78 features 
Apply MR feature selection with missing threshold= 0.3
5 features with greater than 0.30 missing values.

List of variables to be removed : ['Alley', 'Fence', 'FireplaceQu', 'MiscFeature', 'PoolQC']
After feature selection:
73 features remain
['LotConfig', 'GarageYrBlt', 'LotShape', 'LowQualFinSF', 'BsmtExposure', 'GarageArea', 'BldgType', 'Condition1', 'Exterior1st', 'HouseStyle', 'LandContour', 'Fireplaces', '2ndFlrSF', 'BsmtFinType2', 'RoofStyle', 'Exterior2nd', 'GarageCond', 'WoodDeckSF', 'BsmtFinType1', 'MasVnrArea', 'PavedDrive', 'OverallQual', 'MSSubClass', 'Utilities', 'Functional', 'BsmtUnfSF', 'CentralAir', 'Foundation', 'YearBuilt', 'MoSold', 'BsmtQual', 'BsmtFinSF1', 'GarageCars', 'MasVnrType', 'ExterCond', 'KitchenQual', 'Electrical', 'OpenPorchSF', 'FullBath', 'Street', 'GarageFinish', 'LotArea', 'BedroomAbvGr', 'LotFrontage', 'HeatingQC', 'YearRemodAdd', 'MSZoning', '3SsnPorch', 'RoofMatl', 'BsmtFullBath', 'Lan[...]<jupyter_text>Learn2clean data preprocessing pipeline  Regression with Learn2Clean<jupyter_code>import learn2clean.loading.reader as rd 
import learn2clean.qlearning.qlearner as ql
import learn2clean.imputation.imputer as imp

# the results of learn2clean cleaning are stored in 'house_example'_results.txt in 'save' directory

d_not_enc = rd.Reader(sep=',',verbose=True, encoding=False) 
house  = ["../datasets/house/house_train.csv","../datasets/house/test.csv"]
dataset= d_not_enc.train_test_split(house, 'SalePrice')
  
# not exhaustively
dataset['train']['YrSold'] = dataset['train']['YrSold'].astype('object')
dataset['train']['YearBuilt'] = dataset['train']['YearBuilt'].astype('object')
dataset['train']['MoSold'] = dataset['train']['MoSold'].astype('object')
dataset['train']['MiscVal'] = dataset['train']['MiscVal'].astype('object')
dataset['train']['GarageYrBlt'] = dataset['train']['GarageYrBlt'].astype('object')
dataset['train']['YearRemodAdd'] = dataset['train']['YearRemodAdd'].astype('object')
dataset['test']

# according to the profiling of House dataset:
#               missing_fraction
# PoolQC                 0.995205
# MiscFeature            0.963014
# Alley                  0.937671
# Fence                  0.807534

# the dataframes train and test will be empty for many leanr2clean episodes
# returning None for accuracy
print(len(dataset['train'].dropna()), len(dataset['test'].dropna()))
# for this particular case, we can impute data first and then run leanr2clean
d1 = imp.Imputer(dataset.copy(), strategy = 'MF',verbose=False).transform()
print(len(d1['train'].dropna()), len(d1['test'].dropna()))

# Learn2clean finds the best strategy 'LC -> LOF -> MARS' for maximal MSE : 0.2232924236591108 for MARS
# in  98.62 seconds
# The best strategy is stored in EOF of 'house_example2_results.txt' in 'save' directory as
# ('house_example2', 'learn2clean', 'MARS', 'SalePrice', None, 'LC -> LOF -> MARS', 'MSE', 0.2232924236591108, 98.62206888198853)

l2c_c1assification1=ql.Qlearner(dataset = d1.copy(),goal='MARS',target_goal='SalePrice',
                                target_prepare=None, file_name = 'house_example2', verbose = False)
l2c_c1assification1.learn2clean()<jupyter_output>Reading csv : house_train.csv ...
Reading data ...
CPU time: 2.574882984161377 seconds
Profiling datasets
        Attribute     Type  Num. Missing Values  Num. Unique Values           Sknewness   Kurtosis
0              Id  float64                  0.0              1460.0                   0       -1.2
1      MSSubClass  float64                  0.0                15.0             1.40621    1.57067
2     LotFrontage  float64                259.0               111.0  2.1608659947055435    17.3753
3         LotArea  float64                  0.0              1073.0             12.1951    202.544
4     OverallQual  float64                  0.0                10.0            0.216721  0.0918565
5     OverallCond  float64                  0.0                 9.0            0.692355    1.09852
6       YearBuilt  float64                  0.0               112.0           -0.612831  -0.442155
7    YearRemodAdd  float64                  0.0                61.0           -0.503044     -1.272
8 [...]<jupyter_text>Classification with Learn2Clean<jupyter_code>import learn2clean.loading.reader as rd 
import learn2clean.qlearning.qlearner as ql
import learn2clean.imputation.imputer as imp

# the results of learn2clean cleaning are stored in 'house_example'_results.txt in 'save' directory

d_not_enc = rd.Reader(sep=',',verbose=True, encoding=True) 
house  = ["../datasets/house/house_train.csv","../datasets/house/test.csv"]
dataset= d_not_enc.train_test_split(house, 'SaleCondition')
  
# not exhaustively
dataset['train']['YrSold'] = dataset['train']['YrSold'].astype('object')
dataset['train']['YearBuilt'] = dataset['train']['YearBuilt'].astype('object')
dataset['train']['MoSold'] = dataset['train']['MoSold'].astype('object')
dataset['train']['MiscVal'] = dataset['train']['MiscVal'].astype('object')
dataset['train']['GarageYrBlt'] = dataset['train']['GarageYrBlt'].astype('object')
dataset['train']['YearRemodAdd'] = dataset['train']['YearRemodAdd'].astype('object')
dataset['test']

# according to the profiling of House dataset:
#               missing_fraction
# PoolQC                 0.995205
# MiscFeature            0.963014
# Alley                  0.937671
# Fence                  0.807534

# the dataframes train and test will be empty for many leanr2clean episodes
# returning None for accuracy
print(len(dataset['train'].dropna()), len(dataset['test'].dropna()))
# for this particular case, we can impute data first and then run leanr2clean
d2 = imp.Imputer(dataset.copy(), strategy = 'MF',verbose=False).transform()
print(len(d1['train'].dropna()), len(d2['test'].dropna()))
# Learn2clean finds the best strategy 'ZSB -> ED -> LDA' for maximal accuracy : 0.8241517694272164 for LDA
# in  4.58 seconds
# The best strategy is stored in EOF of 'house_example2_results.txt' in 'save' directory as
# ('house_example2', 'learn2clean', 'LDA', 'SaleCondition', None, 'ZSB -> ED -> LDA', 'accuracy', 0.8241517694272164, 4.3612353801727295)
l2c_c1assification1=ql.Qlearner(dataset = d2,goal='LDA',target_goal='SaleCondition',
                                target_prepare=None, file_name = 'house_example2', verbose = False)
l2c_c1assification1.learn2clean()

# Learn2clean finds the best strategy WR -> IQR -> NB for maximal accuracy : 0.6120754716981132 for NB
# in  4.31 seconds
# The best strategy is stored in EOF of 'house_example2_results.txt' in 'save' directory as
# ('house_example2', 'learn2clean', 'NB', 'SaleCondition', None, 'WR -> IQR -> NB', 'accuracy', 0.6120754716981132, 4.3143150806427)

l2c_c1assification1=ql.Qlearner(d2,goal='NB',target_goal='SaleCondition',
                                target_prepare=None, file_name = 'house_example2', verbose = False)
l2c_c1assification1.learn2clean()<jupyter_output>Start Learn2Clean
Learn2Clean - Pipeline construction -- CPU time: 0.1809401512145996 seconds
=== Start Pipeline Execution ===


Strategy# 0 : Greedy traversal for starting state DS
DS -> IQR -> NB

Start pipeline
-------------
>>Normalization 
* For train dataset
... train dataset
* For test dataset
... test dataset
Normalization done -- CPU time: 0.15661215782165527 seconds


>>Outlier detection and removal:
* For train dataset
0 outlying rows have been removed
* For test dataset
0 outlying rows have been removed
Outlier detection and removal done -- CPU time: 0.06927299499511719 seconds


>>Classification task
Accuracy of Naive Naive Bayes classification for 10 cross-validation : 0.06783144912641316

Classification done -- CPU time: 0.10594511032104492 seconds
End Pipeline CPU time: 0.3321809768676758 seconds


Strategy# 1 : Greedy traversal for starting state MM
MM -> AD -> NB

Start pipeline
-------------
>>Normalization 
* For train dataset
... train dataset
* For test dataset
..[...]<jupyter_text>Random data preprocessing pipelines<jupyter_code>import learn2clean.loading.reader as rd 
import learn2clean.qlearning.qlearner as ql

# the results of random cleaning are stored in 'house_example2'_results_file.txt in 'save' directory
# appended to the EOF 

#random preprocessing pipeline for LASSO regression
random1=ql.Qlearner(d1,goal='LASSO',target_goal='SalePrice',target_prepare=None, verbose = False)
random1.random_cleaning('house_example2')

#random preprocessing pipeline for OLS regression
random2=ql.Qlearner(d1,goal='OLS',target_goal='SalePrice',target_prepare=None, verbose = True)
random2.random_cleaning('house_example2')

#random preprocessing pipeline for MARS regression
random3=ql.Qlearner(d1,goal='MARS',target_goal='SalePrice',target_prepare=None, verbose = False)
random3.random_cleaning('house_example2')


# no preprocessing for regression: results appended to the EOF 'house_example2'_results.txt 

no_prep1=ql.Qlearner(d1,goal='LASSO',target_goal='SalePrice',target_prepare=None, verbose = False)
no_prep1.no_prep('house_example2')

no_prep2=ql.Qlearner(d1,goal='OLS',target_goal='SalePrice',target_prepare=None, verbose = False)
no_prep2.no_prep('house_example2')

no_prep3=ql.Qlearner(d1,goal='MARS',target_goal='SalePrice',target_prepare=None, verbose = False)
no_prep3.no_prep('house_example2')
# no preprocessing for classification: results appended to the EOF 'house_example2'_results.txt 

no_prep1=ql.Qlearner(d2,goal='CART',target_goal='SaleCondition',target_prepare=None, verbose = False)
no_prep1.no_prep('house_example2')

no_prep2=ql.Qlearner(d2,goal='LDA',target_goal='SaleCondition',target_prepare=None, verbose = False)
no_prep2.no_prep('house_example2')

no_prep3=ql.Qlearner(d2,goal='NB',target_goal='SaleCondition',target_prepare=None, verbose = False)
no_prep3.no_prep('house_example2')<jupyter_output>Start pipeline
-------------

>>Classification task
Avg accuracy of CART classification for 10 cross-validation : 0.8828365878725591

Classification done -- CPU time: 99.2328782081604 seconds
End Pipeline CPU time: 99.23292994499207 seconds

Start pipeline
-------------

>>Classification task

Accuracy of LDA result for 10 cross-validation : 0.8074683110654334

Classification done -- CPU time: 0.09183597564697266 seconds
End Pipeline CPU time: 0.09188604354858398 seconds

Start pipeline
-------------

>>Classification task
Accuracy of Naive Naive Bayes classification for 10 cross-validation : 0.10551558752997602

Classification done -- CPU time: 0.12027192115783691 seconds
End Pipeline CPU time: 0.12030696868896484 seconds
{'fraction_non_alphanumeric': 0.11578076836328095, 'fraction_numerical': 0.088364283387214, 'mean_word_length': 3.70067145717988, 'pattern_counts': {'":': 4, '<': 47, '<?xml version=': 0, '>': 123, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 5, 'word_list_counts': {'cursed_substrings.json': 62, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '647801', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 14886, 'n_tokens_neox': 12407, 'n_words': 3322, 'path': 'examples/Learn2Clean-examples-house.ipynb', 'repo_name': 'yuancz/Learn2Clean', 'size': '39903'}