<jupyter_start><jupyter_text>Imports<jupyter_code>import re
import pandas as pd
import numpy as np
import unidecode

import seaborn as sns
import matplotlib.pyplot as plt

# Silence warnings
import warnings
warnings.filterwarnings("ignore")
pd.options.mode.chained_assignment = None<jupyter_output><empty_output><jupyter_text>Basic morphological stats<jupyter_code>def count_words(text):
    ''' Returns the word (i.e. type) count for a given text.
    
    Input:
        text(str)
    
    Output:
        count_words(pandas DataFrame): return the count for each word
    '''
        
    words = re.split('[\s\.,·:;\'—«»*’]', text)
    count_words = pd.Series(words).value_counts()
    
    count_words = pd.DataFrame(count_words).rename(columns = {0:'count'})
    
    length = []
    
    for word in count_words.index:
        length.append(len(word))
        
    count_words['length'] = length
    
    return count_words
def read_txt(name):
    
    f = open("txt/" + name + ".txt", "r")
    text = f.read()
    f.close()
    
    return text
def total_words(text):
    
    words = re.split('[\s\.,·:;\'—«»*’]', text)
    total = []
    
    for word in words:
        if len(word) > 0:
            total.append(word)
            
    return len(total)
def get_metrics(text):
    
    print(len(text), 'total characters,',
          total_words(text), 'total words,',
          len(count_words(text)), 'unique words')
lycurgus_lem = read_txt('lycurgus_lem')
get_metrics(lycurgus_lem)
lycurgus = read_txt('lycurgus')
get_metrics(lycurgus)
hesiode_lem = read_txt('hesiode_lem')
get_metrics(hesiode_lem)
hesiode = read_txt('hesiode')
get_metrics(hesiode)
aristides_lem = read_txt('aristides_lem')
get_metrics(aristides_lem)
aristides = read_txt('aristides')
get_metrics(aristides)
aristotle_lem = read_txt('aristotle_lem')
get_metrics(aristotle_lem)
aristotle = read_txt('aristotle')
get_metrics(aristotle)
antiphon_lem = read_txt('antiphon_lem')
get_metrics(antiphon_lem)
antiphon = read_txt('antiphon')
get_metrics(antiphon)
thucydides_lem = read_txt('thucydides_lem')
get_metrics(thucydides_lem)
thucydides = read_txt('thucydides')
get_metrics(thucydides)
iliad_Homer_lem = read_txt('iliad_Homer_lem')
get_metrics(iliad_Homer_lem)
iliad_Homer = read_txt('iliad_Homer')
get_metrics(iliad_Homer)
odyssey_Homer_lem = read_txt('odyssey_Homer_lem')
get_metrics(odyssey_Homer_lem)
odyssey_Homer = read_txt('odyssey_Homer')
get_metrics(odyssey_Homer)
iliad_RL = read_txt('iliad_RL')
get_metrics(iliad_RL)
iliad_KK = read_txt('iliad_KK')
get_metrics(iliad_KK)
oral = read_txt('oral')
get_metrics(oral)
oral_lem = read_txt('oral_lem')
get_metrics(oral_lem)
written = read_txt('written')
get_metrics(written)
written_lem = read_txt('written_lem')
get_metrics(written_lem)<jupyter_output>1290772 total characters, 200096 total words, 24985 unique words<jupyter_text>Type-token ratio<jupyter_code>def type_token_ratio(text, tokens = 1000):
    ''' Compute the type-token ratio (TTK) of a text, on n samples of same token length.
    
    Inputs:
        text(str): text/corpus
        n_sample(int): number of repeated measures
        tokens(int): number of tokens (words) of each TTK measure
    
    Output:
        type_token_ratio(list<float>): list of the TTK measures
    '''
    
    words_ = re.split('[\s\.,·:;\'—«»*’]', text)
    words, type_token_ratio = [], []
    
    for word_ in words_:
        if word_ != '':
            words.append(word_)
            
    n_sample = len(words) // tokens
    
    for n in range(n_sample):
        
        sample = words[n: n+tokens]
        
        text = ''
        for word in sample:
            text += word + ' '
                    
        text = text[:-1]
        
        type_ = len(count_words(text)) - 1
        type_token_ratio.append(type_ / tokens)
        
    return type_token_ratio
TTR_iliad_Homer = type_token_ratio(iliad_Homer, tokens = 500)
TTR_iliad_KK = type_token_ratio(iliad_KK, tokens = 500)
TTR_iliad_RL = type_token_ratio(iliad_RL, tokens = 500)
sns.set(rc={'figure.figsize':(20,15)})
sns.set_style("ticks")
sns.set_context("paper", font_scale=2)                                                  
plt.boxplot([TTR_iliad_Homer, TTR_iliad_KK, TTR_iliad_RL], notch = True);

plt.xticks([1,2,3], np.array(['Homer version \n (ancient greek)', 
                              'Kazantzakis-Kakridis version \n (modern greek)', 
                              'Lattimore version \n (english)']))
plt.ylabel('TTR');
plt.ylim((.45,.8))
plt.title('Type Token Ratio of the Iliad by version (500 tokens slices) \n');
plt.show()
TTR_Homer = type_token_ratio(iliad_Homer + odyssey_Homer, tokens = 500)
TTR_hesiode = type_token_ratio(hesiode, tokens = 500)
TTR_lycurgus = type_token_ratio(lycurgus, tokens = 500)
plt.boxplot([TTR_Homer, TTR_hesiode, TTR_lycurgus], notch = True);

plt.xticks([1,2,3], np.array(['Homer \n (oral hexametric poetry, 8th BC)',
                                'Hesiode \n (written hexametric poetry, 8th BC)', 
                                'Lycurgus \n (speech, 8th BC)']))
plt.ylabel('TTR');
plt.ylim((.45,.8))
plt.title('Type Token Ratio in 8th century work (500 tokens slices) \n ');
plt.show()
TTR_oral = type_token_ratio(oral_lem, tokens = 500)
TTR_written = type_token_ratio(written_lem, tokens = 500)
plt.boxplot([TTR_Homer, TTR_written, TTR_oral], notch = True);

plt.xticks([1,2,3], np.array(['Homeric greek',
                             'Written greek',
                             'Oral greek']))
plt.ylabel('TTR');
plt.ylim((.45,.8))
plt.title('Type Token Ratio in Ancient greek (lemmatized, 500 tokens slices) \n ');
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1289469042159111, 'fraction_numerical': 0.015170224025401306, 'mean_word_length': 3.402173913043478, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '470133', 'license': 'MIT', 'n_tokens_mistral': 2160, 'n_tokens_neox': 2045, 'n_words': 406, 'path': 'TTR.ipynb', 'repo_name': 'RPetitpierre/Homeric_epics_quantitative_analysis', 'size': '5669'}