<jupyter_start><jupyter_text>______________________<jupyter_code>df_sample_ans = pd.read_csv("../input/sample_submission.csv")
df_sample_ans.head()<jupyter_output><empty_output><jupyter_text>__________________<jupyter_code>df_test = pd.read_csv("../input/test.csv")
df_test.head()
df_my_const_ans = pd.DataFrame()
df_my_const_ans["id"] = df_test["id"].values

for i, name in enumerate(y_names):
    df_my_const_ans[name] = np.ones(len(df_test)) * const_answer[i]
df_my_const_ans.head()
df_my_const_ans.to_csv("my_const_ans.csv", index=False)
!cat my_const_ans.csv | head -n 5
!cat sample_submission.csv | head -n 5<jupyter_output>cat: sample_submission.csv: No such file or directory<jupyter_text>_____________________<jupyter_code>df_my_const_ans_2 = pd.DataFrame()
df_my_const_ans_2["id"] = df_test["id"].values

for i, name in enumerate(y_names):
    df_my_const_ans_2[name] = np.zeros(len(df_test))

def save_prediction(df, name):
    df.to_csv(name, index=False)
    
save_prediction(df_my_const_ans_2, "my_const_ans_2.csv")
!cat my_const_ans_2.csv | head -n 5<jupyter_output>id,toxic,severe_toxic,obscene,threat,insult,identity_hate
00001cee341fdb12,0.0,0.0,0.0,0.0,0.0,0.0
0000247867823ef7,0.0,0.0,0.0,0.0,0.0,0.0
00013b17ad220c46,0.0,0.0,0.0,0.0,0.0,0.0
00017563c3f7919a,0.0,0.0,0.0,0.0,0.0,0.0
cat: write error: Broken pipe<jupyter_text>____________________________________<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
%%time
vectorizer = TfidfVectorizer(max_features=10000, stop_words="english",ngram_range=(1,4))
vectorizer.fit(df["comment_text"])
%%time
test_v = vectorizer.transform(df["comment_text"])
%%time
train_v = vectorizer.transform(df_test["comment_text"])
from sklearn.linear_model import LogisticRegression
import tqdm
df_prediction = pd.DataFrame()
df_prediction["id"] = df_test["id"].values

for name in tqdm.tqdm(y_names):
    LR = LogisticRegression()
    LR.fit(test_v, df[name].values)
    df_prediction[name] = LR.predict_proba(train_v)[:, 1]
    # print(LR.predict_proba(train_v)[:5])
df_prediction.head()
save_prediction(df_prediction, "tfid_logreg_prediction_10k.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14364896073903002, 'fraction_numerical': 0.05404157043879908, 'mean_word_length': 8.06276150627615, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '349194', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 954, 'n_tokens_neox': 906, 'n_words': 106, 'path': 'ml/toxic.ipynb', 'repo_name': 'shevkunov/workout', 'size': '2165'}