<jupyter_start><jupyter_text>Nice version<jupyter_code>class DataframeContainer:
    def __init__(self, name):
        self.name = name
        self.dataframe = pd.read_csv('dataset/train_all.csv', sep=';')
        
    def filter_dataframe(self):
        count = 0
        for ind, row in self.dataframe.iterrows():
            if self.name != str(row['Label']):
                count += 1
                row['Label'] = 'General'
                row['Text'] = row['Text'].replace('\n', ' ').replace(',', ' ').lower()
        print(f'{self.name} filtered {count} rows')  

    def separate_x_y(self):
        self.df_X, self.df_y = self.dataframe['Text'], self.dataframe['Label']
        unique, counts = np.unique(self.df_y , return_counts=True)
        plt.bar(unique, counts, 1)
        plt.title('Class Frequency')
        plt.xlabel('Class')
        plt.ylabel('Frequency')
        plt.show()
        
    def split_train_test(self, test_size = 0.3, random_state = 42):
        self.X_train, self.X_test, self.y_train, self.y_test = train_test_split(self.df_X, self.df_y, test_size=test_size, random_state=random_state, stratify=self.df_y)

    def clf_fit(self):
        self.clf = Pipeline([
            ('vectorizer', CountVectorizer(analyzer="word",
                                           tokenizer=word_tokenize,
                                           max_features=None,
                                           lowercase=True)),
            #('smote', SMOTE(random_state=42)),
            ('classifier', LinearSVC(random_state=42))
        ])
        self.clf.fit(self.X_train, self.y_train)
        print(f'{self.name} clf fit done')
        
    def predict(self):
        self.y_pred = self.clf.predict(self.X_test)

    def confusion_matrix(self):
        y_unique = self.y_test.unique()
        cm = confusion_matrix(self.y_test, self.y_pred, labels=y_unique)
        cm_df = pd.DataFrame(cm, index = [y_unique], columns = [y_unique])
        plt.figure(figsize=(5,4))
        sns.heatmap(cm_df, annot=True, fmt='d')
        plt.title('Confusion Matrix')
        plt.ylabel('Actual Values')
        plt.xlabel('Predicted Values')
        plt.show()
        print(f"Accuracy {self.name} : {metrics.accuracy_score(self.y_test, self.y_pred)}")
    def save_pickle(self):
        filename = self.name + '.sav'
        pickle.dump(self.clf, open(filename, 'wb'))
names_list = ["Audio", "Computer Vision", "Graphs", "Natural Language Processing", "Reinforcement Learning", "Sequential"]
#names_list = ["Graphs"]
dataframecontainers_list = [DataframeContainer(name) for name in names_list]
for container in dataframecontainers_list:
    container.filter_dataframe()
    container.separate_x_y()
    container.split_train_test()
for container in dataframecontainers_list:
    container.clf_fit()
for container in dataframecontainers_list:
    container.predict()
    container.confusion_matrix()
for container in dataframecontainers_list:
    container.save_pickle()
def finaltest(df_train_all):
    pos = 0
    df_train_all_x = df_train_all['Text']
    for i in range(len(df_train_all_x)):
        df_train_all_x[i] = df_train_all_x[i].replace('\n', ' ').replace(',', ' ').lower()
    df_train_all_y = df_train_all['Label']
    for x in range(df_train_all_x.shape[0]):
        X = df_train_all_x.iloc[[x]]
        y = df_train_all_y.iloc[x]
        #predictions = [dataframe.clf.predict(X)[0] for dataframe in dataframecontainers_list]
        for dataframecontainer in dataframecontainers_list:
            #print(dataframecontainer.name)
            prediction = dataframecontainer.clf.predict(X)[0]
         #   p.append(prediction)
            if prediction != 'Other' and prediction == y:
                pos += 1
                break
        #print(p, y)
    print(pos/df_train_all_x.shape[0])
        
finaltest(df_train_all)<jupyter_output>0.9984512931702029
{'fraction_non_alphanumeric': 0.12966297916130692, 'fraction_numerical': 0.00926164136866478, 'mean_word_length': 2.6507042253521127, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '160023', 'license': 'Apache-2.0', 'n_tokens_mistral': 1298, 'n_tokens_neox': 1255, 'n_words': 221, 'path': 'documentation/olds/aimmx_reproduction-general.ipynb', 'repo_name': 'kuefmz/software_classification', 'size': '3887'}