<jupyter_start><jupyter_text>Featurefilter ExamplesThis notebooks shows a number of concise examples for common use cases.  Remove columns with too many NA values<jupyter_code>import numpy as np
import pandas as pd

from featurefilter import NaFilter

df = pd.DataFrame({'A': [0, np.nan, np.nan],
                   'B': [0, 0, np.nan]})

na_filter = NaFilter(max_na_ratio=0.5)
na_filter.columns_to_drop = ['A']
na_filter.fit_transform(df)<jupyter_output>The NA ratio of column 'A' (0.6667) is above the threshold of 0.5000<jupyter_text>Remove columns with too low or high variance<jupyter_code>import pandas as pd

from featurefilter import VarianceFilter

df = pd.DataFrame({'A': [0., 1.], 'B': [0., 0.]})

variance_filter = VarianceFilter()
variance_filter.fit_transform(df)<jupyter_output>The variance of column 'B' (0.0000) is below the threshold of 0.0000<jupyter_text>Remove columns with too high correlation to the target variables<jupyter_code>import pandas as pd

from featurefilter import TargetCorrelationFilter

df = pd.DataFrame({'A': [0, 0], 'B': [0, 1], 'Y': [0, 1]})

target_correlation_filter = TargetCorrelationFilter(target_column='Y')
target_correlation_filter.fit_transform(df)<jupyter_output>The absolute correlation of column 'B' (1.0000) to the target column 'Y' is above the threshold of 0.9500<jupyter_text>Remove columns using generalized linear models (GLMs)<jupyter_code>import pandas as pd

from featurefilter import GLMFilter

df = pd.DataFrame({'A': [0, 0, 1, 1],
                   'B': [0, 1, 0, 1],
                   'Y': [0, 0, 1, 1]})

glm_filter = GLMFilter(target_column='Y', top_features=1)
glm_filter.fit_transform(df)<jupyter_output>Relative feature importances:
   A 1.0000
   B 0.0000<jupyter_text>Remove columns using tree-based models<jupyter_code>import pandas as pd

from featurefilter import TreeBasedFilter

df = pd.DataFrame({'A': [0, 0, 1, 1],
                   'B': [0, 1, 0, 1],
                   'Y': ['a', 'a', 'b', 'b']})

tree_based_filter = TreeBasedFilter(target_column='Y',
                                    categorical_target=True,
                                    top_features=1)
tree_based_filter.fit_transform(df)<jupyter_output>A 1.0
B 0.0
['B' 'A']<jupyter_text>Remove columns using multiple filters combined with scikit-learn's Pipeline API<jupyter_code>import numpy as np
import pandas as pd
from sklearn.pipeline import Pipeline

from featurefilter import NaFilter, VarianceFilter

df = pd.DataFrame({'A': [0, np.nan, np.nan],
                   'B': [0, 0, 0],
                   'C': [0, np.nan, 1]})

pipeline = Pipeline([
    ('na_filter', NaFilter(max_na_ratio=0.5)),
    ('variance_filter', VarianceFilter())
])

pipeline.fit_transform(df)<jupyter_output>The NA ratio of column 'A' (0.6667) is above the threshold of 0.5000
The variance of column 'B' (0.0000) is below the threshold of 0.0000<jupyter_text>Remove columns using existing selectors provided by scikit-learn<jupyter_code>import pandas as pd
from sklearn.feature_selection import RFECV
from sklearn.linear_model import LinearRegression

from featurefilter import SklearnWrapper

df = pd.DataFrame({'A': [0, 0, 1, 1],
                   'B': [0, 1, 0, 1],
                   'Y': [0, 0, 1, 1]})

model = RFECV(LinearRegression(),
              min_features_to_select=1,
              cv=3)
selector = SklearnWrapper(model, target_column='Y')
selector.fit_transform(df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11788617886178862, 'fraction_numerical': 0.03571428571428571, 'mean_word_length': 3.9074074074074074, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '463713', 'license': 'MIT', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1139, 'n_words': 329, 'path': 'examples.ipynb', 'repo_name': 'floscha/featurefilter', 'size': '3444'}