<jupyter_start><jupyter_text><jupyter_code>import string
import time

import numpy as np
import pandas as pd
from collections import Counter

import seaborn as sns
import matplotlib.pyplot as plt

import nltk
from nltk.stem.porter import PorterStemmer
data = pd.read_csv('/tmp/datasetNew.csv', encoding='utf8')

print(data.head())

text = list(data['TEXT'].values)
author = list(data['AUTHOR'].values)

print('Dataset contains {} notes.'.format(len(text)))
authors = Counter(author)
authors
author_names = list(authors.keys())
author_names
np.random.seed(73)
n = len(text)

for _ in range(5):
    print(text[np.random.randint(0, n)])
 print(type(text)) 

word_count = np.array([len(sent.split()) for sent in data['TEXT'].astype(str)])

char_count = np.array([len(sent) for sent in data['TEXT'].astype(str)])
ave_length = char_count / word_count
def get_stats(var):    
    print('\t Min: ', np.min(var))
    print('\t Max: ', np.max(var))
    print('\t Average: ', np.mean(var))
    print('\t Median: ', np.median(var))
    print('\t Percentile 1%: ', np.percentile(var, 1))
    print('\t Percentile 95%: ', np.percentile(var, 95))
    print('\t Percentile 99%: ', np.percentile(var, 99))
    print('\t Percentile 99.5%: ', np.percentile(var, 99.5))
    print('\t Percentile 99.9%: ', np.percentile(var, 99.9))
print('Word count statistics:')
get_stats(word_count)
sns.distplot(word_count, kde=True, bins=80, color='green').set_title('Distribution of word count')
plt.xlabel('Sentence length in words')
plt.ylabel('Number of offers')
plt.xlim(0, 100)
plt.savefig('word_count.png')
print('Character count statistics:')
get_stats(char_count)
sns.distplot(char_count, kde=True, bins=80, color='green').set_title('Distribution of characters')
plt.xlabel('Sentence length in characters')
plt.ylabel('Number of sentences')
plt.xlim(0, 400)
plt.savefig('char_count.png')
print('Average length statistics:')
get_stats(ave_length)
sns.distplot(ave_length, kde=True, bins=80, color='green').set_title('Distribution of average word length')
plt.xlabel('Average word length in characters')
plt.ylabel('Number of sentences')
plt.xlim(0, 10)
plt.savefig('ave_length.png')
word_outliers = np.where(word_count > 150)

for i in word_outliers[0][:5]:
    print('Author: {}, Sentence length: {}'.format(author[i], word_count[i]))
    print(text[i], '\n')
max_authors = {author : 0 for author in author_names}

for i in word_outliers[0]:
    max_authors[author[i]] += 1

Counter(max_authors)
word_outliers = np.where(word_count < 2)

for i in word_outliers[0][:10]:
    print('Sentence length: {}'.format(word_count[i]))
    print(text[i], '\n')
text_string = ''
for sents in data['TEXT'].astype(str):
    text_string += sents.lower()

char_cnt = Counter(text_string)
print(char_cnt)
print(len(char_cnt), 'unusual symbols in data.')
print(list(char_cnt.keys()))
accented_chars = ['f', 'u', 'r', 's', 'i', 'c', 'h', '́', 'n', 'd', 'p', 'e', 'a', 't', 'o', 'l', 'x', 'm', 'j', 'é', 'ô', 'v', 'q', 'ê', 'g', 'b', 'k', 'y', 'à', 'і', 'z', 'w', 'è', 'ó', 'ö', '°', 'ç', 'ï', 'á', 'ü', 'ù', 'û', 'î', 'ѣ', 'â']

accented_text = []
for i in range(len(text)):
    for j in data['TEXT'].astype(str)[i]:
        if j in accented_chars:
            accented_text.append(i)
        
accented_text = list(set(accented_text))
 
print(len(accented_text), 'sentences contains unusual symbols.')
for i in accented_text[:10]:
    print('Sentence number {}: '.format(i))
    print(text[i], '\n')
text = [excerpt.replace('\xa0', '').replace('\x7f', '') for excerpt in data['TEXT'].astype(str)]
ctr = 0
for excerpt in text:
    if '  ' in excerpt:
        ctr += 1

print(ctr, 'occurrences of large blocks of indentation.')
new_text = []
for excerpt in text:
    while '  ' in excerpt:
        excerpt = excerpt.replace('  ',' ')
    new_text.append(excerpt)

text = new_text
print(len(text))
normed_text = []

for sent in text:
    new = sent.lower()
    new = new.translate(str.maketrans('','', string.punctuation))
    new = new.replace('“', '').replace('”', '') # english quotes
    new = new.replace('‟', '').replace('”', '') # french quotes
    new = new.replace('«', '').replace('»', '') # christmas tree quotes
    new = new.replace('—', '').replace('–', '') # em dash
    new = new.replace('(', '').replace(')', '')
    new = new.replace('…', '') # ellipsis as one character
    
    normed_text.append(new)
    
print(normed_text[0:5])
print(len(normed_text))
data['TEXT'] = normed_text

data.to_csv('preprocessed_data.csv', index=False)
from typing import List

import numpy as np
import pandas as pd
from collections import Counter
import seaborn as sns
import matplotlib.pyplot as plt
import string
import time

from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.metrics import precision_recall_fscore_support as score
from sklearn.preprocessing import LabelBinarizer

import keras
from keras.models import Model
from keras.layers import Input, Dense, Flatten, Dropout, Embedding
from keras.layers.convolutional import Conv1D, MaxPooling1D
from keras.layers.merge import concatenate
from keras.optimizers import Adam
from keras.preprocessing.text import one_hot
from keras.callbacks import ModelCheckpoint 

from scipy import stats
data = pd.read_csv("preprocessed_data.csv", encoding='utf8')
print(data.head())
normed_text = list(data['TEXT'])
author = list(data['AUTHOR'])

authors_names = list(Counter(author).keys())
authors_count = len(authors_names)

normed_text = [str(i) for i in normed_text]

def plot_confusion_matrix(cm, classes: List[str],
                          normalize: bool = False,
                          title: str = 'Confusion matrix',
                          cmap = plt.cm.Greens):
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print('Normalized confusion matrix')
    else:
        print('Unnormalized confusion matrix')

    print(cm)
       
    df_cm = pd.DataFrame(cm, index = classes,
                  columns = classes)
    sns.heatmap(df_cm, annot=True, cmap = cmap)
    plt.ylabel('Right author')
    plt.xlabel('Predicted author')
    plt.title(title)
def plot_history_of_accurancy(history):
    plt.plot(history.history['accuracy'])
    plt.plot(history.history['val_accuracy'])
    plt.title('Model\'s accurancy')
    plt.ylabel('accurancy')
    plt.xlabel('epochs')
    plt.legend(['teaching data', 'test data'], loc='upper left')
def plot_history_of_loss(history):
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('Model\'s error')
    plt.ylabel('error')
    plt.xlabel('epochs')
    plt.legend(['teaching data', 'test data'], loc='upper left')
text_train, text_test, author_train, author_test = train_test_split(normed_text, author, test_size=0.2, random_state=5)
print(np.shape(text_train))
print(np.shape(text_test))
print(np.shape(author_train))
print(np.shape(author_test))
def create_n_grams(excerpt_list: List[str], n: int, vocab_size: int, seq_size: int):
    n_gram_list = []

    for excerpt in excerpt_list:
        excerpt = excerpt.replace(" ", "")

        n_grams = [excerpt[i:i + n] for i in range(len(excerpt) - n + 1)]

        new_string = " ".join(n_grams)

        hot = one_hot(new_string, round(vocab_size * 1.3))

        hot_len = len(hot)
        if hot_len >= seq_size:
            hot = hot[0:seq_size]
        else:
            diff = seq_size - hot_len
            extra = [0]*diff
            hot = hot + extra

        n_gram_list.append(hot)
    
    n_gram_array = np.array(n_gram_list)
    
    return n_gram_array
def get_vocab_size(excerpt_list: List[str], n: int, seq_size: int) -> int:
    n_gram_list = []

    for excerpt in excerpt_list:
        excerpt = excerpt.replace(" ", "")
   
        n_grams = [excerpt[i:i + n] for i in range(len(excerpt) - n + 1)]

        gram_len = len(n_grams)
        if gram_len >= seq_size:
            n_grams = n_grams[0:seq_size]
        else:
            diff = seq_size - gram_len
            extra = [0]*diff
            n_grams = n_grams + extra
        
        n_gram_list.append(n_grams)
    
    n_gram_list = list(np.array(n_gram_list).flat)
    
    n_gram_cnt = Counter(n_gram_list)
    vocab_size = len(n_gram_cnt)
    
    return vocab_size
vocab_sizes = []
for i in range(1, 4):
    vocab_sizes.append(get_vocab_size(text_train, i, 730))
    print('Size for n =', i, 'is:', vocab_sizes[i - 1])
gram1_train = create_n_grams(text_train, 1, vocab_sizes[0], 730)
gram2_train = create_n_grams(text_train, 2, vocab_sizes[1], 730)
gram3_train = create_n_grams(text_train, 3, vocab_sizes[2], 730)
gram1_test = create_n_grams(text_test, 1, vocab_sizes[0], 730)
gram2_test = create_n_grams(text_test, 2, vocab_sizes[1], 730)
gram3_test = create_n_grams(text_test, 3, vocab_sizes[2], 730)
print(np.shape(gram1_train))
print(np.shape(gram2_train))
print(np.shape(gram3_train))

print(np.shape(gram1_test))
print(np.shape(gram2_test))
print(np.shape(gram3_test))
max_1gram = np.max(gram1_train)
max_2gram = np.max(gram2_train)
max_3gram = np.max(gram3_train)

print('Max value for 1-gramms: ', max_1gram)
print('Max value for bigramms: ', max_2gram)
print('Max value for trigramms: ', max_3gram)
processed_train = text_train
processed_test = text_test

print(processed_train[0:5])
vectorizer = TfidfVectorizer(strip_accents = 'unicode', min_df = 6)
vectorizer.fit(processed_train)

print('Dictionary size: ', len(vectorizer.vocabulary_))

words_train = vectorizer.transform(processed_train)
words_test = vectorizer.transform(processed_test)
print(type(author_train))
print(len(author_train))
author_lb = LabelBinarizer()

author_lb.fit([str(i) for i in author_train])
author_train_hot = author_lb.transform([str(i) for i in author_train])
author_test_hot = author_lb.transform([str(i) for i in author_test])

print(author_train_hot)

#from keras.utils import to_categorical

#author_train_hot = to_categorical(author_train_hot)
# change to [1, 0, 0,..., 0]  --> [[0, 1], [1, 0], ..., [1, 0]]
print(type(author_train_hot))
def define_model(input_len: int, output_size: int, vocab_size : int, embedding_dim: int, verbose: bool = True,
                drop_out_pct: float = 0.25, conv_filters: int = 500, activation_fn: str = 'relu', pool_size: int = 2, learning: float = 0.0001):
    inputs1 = Input(shape=(input_len,))
    embedding1 = Embedding(vocab_size, embedding_dim)(inputs1)
    drop1 = Dropout(drop_out_pct)(embedding1)
    conv1 = Conv1D(filters=conv_filters, kernel_size=3, activation=activation_fn)(drop1)
    pool1 = MaxPooling1D(pool_size=pool_size)(conv1)
    flat1 = Flatten()(pool1)
    
    inputs2 = Input(shape=(input_len,))
    embedding2 = Embedding(vocab_size, embedding_dim)(inputs2)
    drop2 = Dropout(drop_out_pct)(embedding2)
    conv2 = Conv1D(filters=conv_filters, kernel_size=4, activation=activation_fn)(drop2)
    pool2 = MaxPooling1D(pool_size=pool_size)(conv2)
    flat2 = Flatten()(pool2)
    
    inputs3 = Input(shape=(input_len,))
    embedding3= Embedding(vocab_size, embedding_dim)(inputs3)
    drop3 = Dropout(drop_out_pct)(embedding3)
    conv3 = Conv1D(filters=conv_filters, kernel_size=5, activation=activation_fn)(drop3)
    pool3 = MaxPooling1D(pool_size=pool_size)(conv3)
    flat3 = Flatten()(pool3)

     
    merged = concatenate([flat1, flat2, flat3])
    
    output = Dense(output_size, activation='softmax')(merged)
    
    model = Model(inputs=[inputs1, inputs2, inputs3], outputs=output)
    
    model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=learning), metrics=['accuracy'])
    
    if verbose:
        print(model.summary())
        
    return model
gram1_model = define_model(730, authors_count, max_1gram + 1, 26)
print(type(authors_count))
print(gram1_train.shape)
gram1_model_history = gram1_model.fit([gram1_train, gram1_train, gram1_train], author_train_hot, epochs=10, batch_size=32, 
                verbose = 1, validation_split = 0.2)
gram2_model = define_model(350, authors_count, max_2gram + 1, 300)
gram2_model_history = gram2_model.fit([gram2_train, gram2_train, gram2_train], author_train_hot, epochs=10, batch_size=32, 
                verbose = 1, validation_split = 0.2)
t0 = time.time()
gram3_model = define_model(350, authors_count, max_3gram + 1, 600)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14497204504291678, 'fraction_numerical': 0.020631545791007167, 'mean_word_length': 3.878985785631963, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '86791', 'license': 'MIT', 'n_tokens_mistral': 4820, 'n_tokens_neox': 4509, 'n_words': 953, 'path': 'Untitled26.ipynb', 'repo_name': 'salhasalman/DatabaseTools', 'size': '12699'}