<jupyter_start><jupyter_text>Automated Essay Scoring- applying NLP and Regressor model to analyse the model- Dataset is from kaggle   importing the library<jupyter_code>import pandas as pd
from sklearn.ensemble import RandomForestRegressor
from sklearn.cross_validation import cross_val_score
from nltk.tokenize import word_tokenize,sent_tokenize
from sklearn.feature_extraction.text import CountVectorizer<jupyter_output>C:\Users\SAMEER\Anaconda3\lib\site-packages\sklearn\ensemble\weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.
  from numpy.core.umath_tests import inner1d
C:\Users\SAMEER\Anaconda3\lib\site-packages\sklearn\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)<jupyter_text>importing the dataset<jupyter_code>train_data = pd.read_table('Data/training_set_rel3.tsv',encoding = "ISO-8859-1")
train_data.head()
train_data_except_set2 = train_data.loc[train_data.essay_set != 2,['essay_set','essay','domain1_score']]
train_data_except_set2.columns = ['essay_set','essay','score']
train_data_set2_1 = train_data.loc[train_data['essay_set'] == 2 ,['essay_set','essay','domain1_score']]
train_data_set2_1['essay_set'] = "2_1"
train_data_set2_1.columns = ['essay_set','essay','score']
train_data_set2_2 = train_data.loc[train_data['essay_set'] == 2 ,['essay_set','essay','domain2_score']]
train_data_set2_2['essay_set'] = "2_2"
train_data_set2_2.columns = ['essay_set','essay','score']
train_data_except_set2.head()
train_data_set2_2.head()
train_data_set2_2.head()
train_data_format = pd.concat([train_data_except_set2,train_data_set2_2,train_data_set2_2])
train_data_format.shape
def get_distinct_word_count(text):
    word_list = word_tokenize(text)
    return len(set(word_list))
#tokenize the word
def get_word_count_token(text):
    word_list = word_tokenize(text)
    return len(word_list)
#tokenize the sentence
def get_sentence_count(text):
    sent_list = sent_tokenize(text)
    return len(sent_list)
import re
def get_character_count(text):
    return len(text)

#removing the space
def get_word_count(text):
    return len(re.findall(r"\s", text))+1
train_data_format['length'] = train_data_format['essay'].apply(get_character_count)
train_data_format['word_count'] = train_data_format['essay'].apply(get_word_count)
train_data_format['distinct_word'] = train_data_format['essay'].apply(get_distinct_word_count)
train_data_format['sentence_count'] = train_data_format['essay'].apply(get_sentence_count)<jupyter_output><empty_output><jupyter_text>Inserting the above line in dataset<jupyter_code>train_data_format.head(5)<jupyter_output><empty_output><jupyter_text>Importing the test set<jupyter_code>test_data = pd.read_table('Data/valid_set.tsv',encoding = "ISO-8859-1")
test_data.head()<jupyter_output><empty_output><jupyter_text>Cleaning the dataset<jupyter_code>test_data_except_set2 = test_data.loc[test_data.essay_set != 2,['essay_set','essay','domain1_predictionid']]
test_data_except_set2.columns = ['essay_set','essay','predictionid']
test_data_set2_1 = test_data.loc[test_data['essay_set'] == 2 ,['essay_set','essay','domain1_predictionid']]
test_data_set2_1['essay_set'] = "2_1"
test_data_set2_1.columns = ['essay_set','essay','predictionid']
test_data_set2_2 = test_data.loc[test_data['essay_set'] == 2 ,['essay_set','essay','domain2_predictionid']]
test_data_set2_2['essay_set'] = "2_2"
test_data_set2_2.columns = ['essay_set','essay','predictionid']
test_data_set2_2.columns
test_data_except_set2.head()
test_data_set2_1.head()
test_data_set2_2.head()
test_data_format = pd.concat([test_data_except_set2,test_data_set2_1,test_data_set2_2])
train_data_format.head()
test_data_format.head()
X = train_data_format['essay']

y = train_data_format['score']

sort_by_id = test_data_format.sort_values(by=['predictionid'], axis=0,inplace=True)
X_test = test_data_format['essay']
test_data_format.shape
X.head()
y.head()<jupyter_output><empty_output><jupyter_text>Convert a collection of text documents to a matrix of token counts<jupyter_code>vect = CountVectorizer()
vect.fit(X)
vect_dtm = vect.transform(X)
vect_dtm<jupyter_output><empty_output><jupyter_text>Applying the random forest<jupyter_code>randomRegressor = RandomForestRegressor(n_estimators=100)
rndfrst = randomRegressor.fit(vect_dtm,y)
test_vect_dtm = vect.transform(X_test)
test_vect_dtm<jupyter_output><empty_output><jupyter_text>Predicting for test case<jupyter_code>y_pred_class = randomRegressor.predict(test_vect_dtm)
y_pred_class_round = [int(i) for i in y_pred_class]
predictionid_list =  [ int(id) for id in test_data_format['predictionid']]<jupyter_output><empty_output><jupyter_text>Saving the predicted data to the new csv file<jupyter_code>pd.DataFrame({'prediction_id':predictionid_list,'predicted_score':y_pred_class_round}).to_csv('Output/submission_2.csv',columns=['prediction_id','predicted_score'],index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11764705882352941, 'fraction_numerical': 0.0196078431372549, 'mean_word_length': 9.893223819301848, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '554619', 'license': 'MIT', 'n_tokens_mistral': 1995, 'n_tokens_neox': 1954, 'n_words': 300, 'path': 'Automated_Essay_Scoring/Automatic_essay_Scoring.ipynb', 'repo_name': 'Akashdeep121/Machine-Learning-Projects', 'size': '5304'}