<jupyter_start><jupyter_text>Pickling<jupyter_code>import _pickle as cPickle
from pathlib import Path

def dumpPickle(fileName, content):
    pickleFile = open(fileName, 'wb')
    cPickle.dump(content, pickleFile, -1)
    pickleFile.close()

def loadPickle(fileName):    
    file = open(fileName, 'rb')
    content = cPickle.load(file)
    file.close()
    
    return content
    
def pickleExists(fileName):
    file = Path(fileName)
    
    if file.is_file():
        return True
    
    return False
trainPickleName = 'pickles/trainWordsDf.pkl'
train = loadPickle(trainPickleName)

devPickleName = 'pickles/devWordsDf.pkl'
dev = loadPickle(devPickleName)
def oneHotEncodeColumns(df):
    columnsToEncode = ['NER', 'POS', "TAG", 'DEP']

    for column in columnsToEncode:
        print(column)
        one_hot = pd.get_dummies(df[column])
        one_hot = one_hot.add_prefix(column + '_')

        df = df.drop(column, axis = 1)
        df = df.join(one_hot)
    
    return df
train = oneHotEncodeColumns(train)
train.head()
dev = oneHotEncodeColumns(dev)
dev.head()
columnsToDrop = ['text', 'titleId', 'paragrapghId', 'sentenceId', 'shape']
train = train.drop(columnsToDrop, axis = 1)
dev = dev.drop(columnsToDrop, axis = 1)
train.head()
dev.head()
#Both dataframe's features
features = []

for feature in dev.columns:
    if feature not in features:
        features.append(feature)
        
for feature in train.columns:
    if feature not in features:
        features.append(feature)
# Add missing features as columns
for feature in features:
    if feature not in train.columns:
        dev[feature] = 0

for feature in features:
    if feature not in dev.columns:
        dev[feature] = 0
x_train = train.drop(labels=['isAnswer'], axis=1)
y_train = train['isAnswer']

x_test = dev.drop(labels=['isAnswer'], axis=1)
y_test = dev['isAnswer']
print(len(x_train))
print(len(y_train))
print(len(x_test))
print(len(y_test))
y_test.head()
from sklearn.naive_bayes import GaussianNB

gnb = GaussianNB()
predictor = gnb.fit(x_train, y_train)
y_pred = predictor.predict(x_test)
correctCount = (y_test == y_pred).sum()

print('Correctly guessed:', '{:.2f}%'.format((correctCount / len(y_test)) * 100))
from sklearn.metrics import confusion_matrix

confusion_matrix(y_test, y_pred)
predictorPickleName = 'pickles/predictor-spacy-features.pkl'
dumpPickle(predictorPickleName, predictor)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12365591397849462, 'fraction_numerical': 0.004962779156327543, 'mean_word_length': 4.202150537634409, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '465195', 'license': 'Apache-2.0', 'n_tokens_mistral': 870, 'n_tokens_neox': 831, 'n_words': 187, 'path': '02 Naive model training.ipynb', 'repo_name': 'KristiyanVachev/fmi-codes-19', 'size': '2418'}