<jupyter_start><jupyter_text>Создание пайплайна с использованием стандартных функций<jupyter_code>preprocessor = Pipeline(steps=[('embeddings', HashingVectorizer()),
#                               ('imputer', SimpleImputer()), 
#                               ('poly', PolynomialFeatures()),
#                               ('log', FunctionTransformer(np.log1p)),
#                               ('scaler', StandardScaler())
                              ])
pipeline = make_pipeline(preprocessor, LogisticRegression(max_iter=10000))

pipeline.fit(data_train.data, data_train.target)

pipeline.score(data_test.data, data_test.target)<jupyter_output><empty_output><jupyter_text>Создание своего класса, преобразующего данные<jupyter_code>class TextTransformer(BaseEstimator, TransformerMixin):
    def __init__(self):
        pass        
    
    def fit(self, X, y = None):
        return self
    
    def transform(self, X , y = None):
        vectorizer = HashingVectorizer()
        X = vectorizer.fit_transform(X)
        return X
X_train, X_test, y_train, y_test = train_test_split(data_train.data, data_train.target,
                                                    test_size=0.3,
                                                    shuffle=True,
                                                    stratify=data_train.target,
                                                    random_state=42)
for data in [X_train, y_train, X_test, y_test]:
    print(len(data))
scores = []

clf = LogisticRegression()
pipeline = make_pipeline(TextTransformer(), clf)
pipeline.fit(X_train, y_train)
scores.append(pipeline.score(data_test.data, data_test.target))
scores
cv = StratifiedKFold(n_compone)<jupyter_output><empty_output><jupyter_text>Пример разных пайплайнов для разных типов полей<jupyter_code>from sklearn import set_config
set_config(display='diagram')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09725158562367865, 'fraction_numerical': 0.005285412262156448, 'mean_word_length': 2.368327402135231, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '15836', 'license': 'MIT', 'n_tokens_mistral': 564, 'n_tokens_neox': 577, 'n_words': 91, 'path': 'Pipelines.ipynb', 'repo_name': 'yurichernyshov/Examples', 'size': '1892'}