<jupyter_start><jupyter_text>Reading Datasets<jupyter_code>train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
sample = pd.read_csv('sample_submission.csv')


pp_train = pp.ProfileReport(train, max_pooling = 6)
# pp_test = pp.ProfileReport(test)

pp_train.to_file('train_analysis.html')
# pp_test.to_file('test_analysis.html')
lgbb = lgb.LGBMClassifier()
lgbb.fit(trainx, trainy)
test.columns
train.columns
train.fillna(train.mean(), inplace=True)
test.fillna(test.mean(), inplace=True)
from scipy.stats import pearsonr
pearsonr(test['Col3'][:15000], train['Col3'][:15000])
the_list = []
for i in range(2, len(train.columns)):
    try:
        if round(pearsonr(test[train.columns[i]][:15000], train[train.columns[i]][:15000])[0], 2) > 0.02:
            print("This is the value for column:", train.columns[i], round(pearsonr(test[train.columns[i]][:15000], train[train.columns[i]][:15000])[0], 2))
            the_list.append(train.columns[i])
            
    except:
        pass
#         print("Couldn't do it for :", train.columns[i])
train[the_list].reset_index()
train_data = train[the_list].values
test_data = test[the_list].values
np.shape(train_data)
lgbb = lgb.LGBMClassifier()
lgbb.fit(train_data, train['Col2'].values)
result = lgbb.predict(test_data)
submission = pd.DataFrame()
submission['Col1'] = test['Col1']
submission['Col2'] = result
submission.to_csv('submission.csv', index=False)
submission.head()
submission['Col2'].value_counts()
train['Col2'].value_counts()
154/20288
1761/15760<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17248062015503876, 'fraction_numerical': 0.041343669250646, 'mean_word_length': 5.853982300884955, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '21170', 'license': 'MIT', 'n_tokens_mistral': 621, 'n_tokens_neox': 582, 'n_words': 86, 'path': '.ipynb_checkpoints/main_code_train-checkpoint.ipynb', 'repo_name': 'shivam13juna/HDFC_HackerEarth_Sep20', 'size': '1548'}