<jupyter_start><jupyter_text>Data Analysis  1. Loading the data<jupyter_code>import re
from nltk import sent_tokenize

cg_sents = []
smg_sents = []

def remove_duplicate_punctuation(s): # sent_tokenize() gets confused when there's duplicate punctuation 
    return(re.sub(r'(\.|\?|!|;)\1+', r'\1 ', s))  # In a previous version of this I used r'\1' instead of r'\1 '. However it caused a problem for sentences like "this is great...I don't know what to do." It seems that some people do not use spacing with ellipses which caused sent_tokenize() to not identify a new sentence.
    
with open('./Data/cg_twitter.txt', 'r', encoding='utf-8') as in_file:
    text = remove_duplicate_punctuation(in_file.read())
    text = re.sub(r'([a-zα-ωίϊΐόάέύϋΰήώ])(\.|\?|;|!)([A-ZΑ-ΩΆΈΊΌΎΏΉ])', r'\1\2 \3', text) # https://stackoverflow.com/questions/44858741/nltk-tokenizer-and-stanford-corenlp-tokenizer-cannot-distinct-2-sentences-withou
    lines = [p for p in text.split('\n') if p] # sent_tokenize() doesn't consider a new line a new sentence so this is required.
    for line in lines:
        cg_sents += sent_tokenize(line)
    
with open('./Data/cg_fb.txt', 'r', encoding='utf-8') as in_file:
    text = remove_duplicate_punctuation(in_file.read())
    text = re.sub(r'([a-zα-ωίϊΐόάέύϋΰήώ])(\.|\?|;|!)([A-ZΑ-ΩΆΈΊΌΎΏΉ])', r'\1\2 \3', text)
    lines = [p for p in text.split('\n') if p]
    for line in lines:
        cg_sents += sent_tokenize(line)
    
with open('./Data/cg_other.txt', 'r', encoding='utf-8') as in_file:
    text = remove_duplicate_punctuation(in_file.read())
    text = re.sub(r'([a-zα-ωίϊΐόάέύϋΰήώ])(\.|\?|;|!)([A-ZΑ-ΩΆΈΊΌΎΏΉ])', r'\1\2 \3', text)
    lines = [p for p in text.split('\n') if p]
    for line in lines:
        cg_sents += sent_tokenize(line)

with open('./Data/smg_twitter.txt', 'r', encoding='utf-8') as in_file:
    text = remove_duplicate_punctuation(in_file.read())
    text = re.sub(r'([a-zα-ωίϊΐόάέύϋΰήώ])(\.|\?|;|!)([A-ZΑ-ΩΆΈΊΌΎΏΉ])', r'\1\2 \3', text)
    lines = [p for p in text.split('\n') if p]
    for line in lines:
        smg_sents += sent_tokenize(line)
    
with open('./Data/smg_fb.txt', 'r', encoding='utf-8') as in_file:
    text = remove_duplicate_punctuation(in_file.read())
    text = re.sub(r'([a-zα-ωίϊΐόάέύϋΰήώ])(\.|\?|;|!)([A-ZΑ-ΩΆΈΊΌΎΏΉ])', r'\1\2 \3', text)
    lines = [p for p in text.split('\n') if p]
    for line in lines:
        smg_sents += sent_tokenize(line)
    
with open('./Data/smg_other.txt', 'r', encoding='utf-8') as in_file:
    text = remove_duplicate_punctuation(in_file.read())
    text = re.sub(r'([a-zα-ωίϊΐόάέύϋΰήώ])(\.|\?|;|!)([A-ZΑ-ΩΆΈΊΌΎΏΉ])', r'\1\2 \3', text)
    lines = [p for p in text.split('\n') if p]
    for line in lines:
        smg_sents += sent_tokenize(line)

cg_sents[:3]<jupyter_output><empty_output><jupyter_text>2. Cleaning the data<jupyter_code>import unicodedata
from string import punctuation
from nltk.tokenize import WhitespaceTokenizer

punctuation += '´΄’…“”–—―»«' # string.punctuation misses these.

def strip_accents(s):
    return ''.join(c for c in unicodedata.normalize('NFD', s)
                   if unicodedata.category(c) != 'Mn')

def get_clean_sent_el(sentence):
    sentence = re.sub(r'^RT', '', sentence)
    sentence = re.sub(r'\&\w*;', '', sentence)
    sentence = re.sub(r'\@\w*', '', sentence)
    sentence = re.sub(r'\$\w*', '', sentence)
    sentence = re.sub(r'https?:\/\/.*\/\w*', '', sentence)
    sentence = ''.join(c for c in sentence if c <= '\uFFFF')
    sentence = strip_accents(sentence)
    sentence = re.sub(r'#\w*', '', sentence)
    sentence = sentence.lower()
    tokens = WhitespaceTokenizer().tokenize(sentence)
    new_tokens = []
    for token in tokens:
        if token == 'ο,τι' or token == 'ό,τι' or token == 'o,ti' or token == 'ó,ti':
            new_tokens.append(token)
        else:
            token = re.sub(r'(?<=[.,!\?;\'΄´’])(?=[^\s])', r' ', token) # If there is punctuation not followed by a space, we add it. I also added a space after apostrophes because I want something like σαγαπώ to not be considered as one word.
            new_token = token.translate(str.maketrans({key: None for key in punctuation}))
            if (new_token != ''): # This might happen if a user surrounds commas with spaces , like so. 
                new_tokens.append(new_token)
    sentence =' '.join(new_tokens)
    sentence = re.sub('\ufeff', '', sentence) # \ufeff might appear when dealing with unicode-encoded files
    sentence = sentence.strip(' ') # performs lstrip() and rstrip()
    sentence = re.sub('  ', ' ', sentence) # Adding a space after the apostrophe can lead to the appearance of double spaces if apostrophes are used along with spaces in the original text.
    return sentence

cg_sents_clean = []
smg_sents_clean = []

for sent in cg_sents:
    cg_sents_clean.append(get_clean_sent_el(sent))
for sent in smg_sents:
    smg_sents_clean.append(get_clean_sent_el(sent))

# Remove empty strings left due to sentences ending up being only URLs then getting deleted on cleaning:
cg_sents_clean = list(filter(None, cg_sents_clean))
smg_sents_clean = list(filter(None, smg_sents_clean))
cg_sents_clean[:3]<jupyter_output><empty_output><jupyter_text>3. Tokenization and setting up to use `nltk.text`  3.1 Tokenization<jupyter_code>cg_sents_tokens = []
smg_sents_tokens = []

for sent in cg_sents_clean:
    cg_sents_tokens.append(WhitespaceTokenizer().tokenize(sent))
for sent in smg_sents_clean:
    smg_sents_tokens.append(WhitespaceTokenizer().tokenize(sent))
    
cg_sents_tokens[:3]<jupyter_output><empty_output><jupyter_text>3.2 Setting up to use `nltk.text`  3.2.1 Words `Text` objects<jupyter_code>from nltk.text import Text

cg_words_flat = [word for sent_tokens in cg_sents_tokens for word in sent_tokens]
smg_words_flat = [word for sent_tokens in smg_sents_tokens for word in sent_tokens]

cg_Text = Text(cg_words_flat)
smg_Text = Text(smg_words_flat)

cg_words_flat = [re.sub(r'ς', 'σ', word) for word in cg_words_flat] # The reason I replace ς with σ for char n-grams is because the Greek final sigma form is arbitrary. No other letter has a final form. When counting the number of sigmas I would like that to include the final forms as well. The σ_ bigram will represent the ς feature so it can be removed.
smg_words_flat = [re.sub(r'ς', 'σ', word) for word in smg_words_flat]

cg_Text<jupyter_output><empty_output><jupyter_text>3.2.2 Word n-grams  3.2.2.1 Bigrams `Text` objects<jupyter_code>from nltk import ngrams

cg_word_bigrams = []
smg_word_bigrams = []

for sent in cg_sents_tokens:
    cg_word_bigrams.append(list(ngrams(sent, 2)))
for sent in smg_sents_tokens:
    smg_word_bigrams.append(list(ngrams(sent, 2)))

print(cg_word_bigrams)    
    
cg_word_bigrams_flat_tuples = [bigram for bigram_list in cg_word_bigrams for bigram in bigram_list]
smg_word_bigrams_flat_tuples = [bigram for bigram_list in smg_word_bigrams for bigram in bigram_list]

cg_word_bigrams_flat = ['%s %s' % bigram_tuple for bigram_tuple in cg_word_bigrams_flat_tuples]
smg_word_bigrams_flat = ['%s %s' % bigram_tuple for bigram_tuple in smg_word_bigrams_flat_tuples]

cg_word_bigrams_Text = Text(cg_word_bigrams_flat)
smg_word_bigrams_Text = Text(smg_word_bigrams_flat)
cg_word_bigrams_Text<jupyter_output>[[('πρασινο', 'αυκουι'), ('αυκουι', 'μες'), ('μες', 'το'), ('το', 'πασχαλινο'), ('πασχαλινο', 'ποτηρι'), ('ποτηρι', 'που'), ('που', 'επιασε'), ('επιασε', 'ο'), ('ο', 'μιτσης')], [('καμνουν', 'πολλα'), ('πολλα', 'ανακαινιση'), ('ανακαινιση', 'στα'), ('στα', 'περβολια')], [('ελα', 'συγγενη'), ('συγγενη', 'τζιαι'), ('τζιαι', 'εχουμε'), ('εχουμε', 'νεοτερα'), ('νεοτερα', 'π'), ('π', 'το'), ('το', 'νικολη')], [('η', 'αληθκεια'), ('αληθκεια', 'με'), ('με', 'τους'), ('τους', 'τερματοφυλακες'), ('τερματοφυλακες', 'και'), ('και', 'οχι'), ('οχι', 'μονο'), ('μονο', 'εχουμε'), ('εχουμε', 'λιο'), ('λιο', 'θεμα')], [('μεχρι', 'και'), ('και', 'αυγουστη'), ('αυγουστη', 'και'), ('και', 'πανο'), ('πανο', 'κωνσταντινου'), ('κωνσταντινου', 'εφεραμε'), ('εφεραμε', 'ρε')], [('ωσπου', 'να'), ('να', 'φαεις'), ('φαεις', 'το'), ('το', 'κραμπι'), ('κραμπι', 'σου'), ('σου', 'εν'), ('εν', 'να'), ('να', 'τελειωσω'), ('τελειωσω', 'μεν'), ('μεν', 'φοασαι')], [('πρεπει', 'να'), ('να', 'εβρω'), ('εβρω', 'αλλη'), ('αλλη[...]<jupyter_text>3.2.3 Chars `Text` objects<jupyter_code>cg_chars_flat = [char for word in cg_words_flat for char in word]
smg_chars_flat = [char for word in smg_words_flat for char in word]

cg_chars_Text = Text(cg_chars_flat)
smg_chars_Text = Text(smg_chars_flat)
cg_chars_Text<jupyter_output><empty_output><jupyter_text>3.2.4 Character n-grams  3.2.4.1 Bigrams `Text` objects<jupyter_code>cg_char_bigrams = []
smg_char_bigrams = []

for word in cg_words_flat:
    cg_char_bigrams.append(list(ngrams(word, 2, pad_left=True, pad_right=True, left_pad_symbol='_', right_pad_symbol='_')))
for word in smg_words_flat:
    smg_char_bigrams.append(list(ngrams(word, 2, pad_left=True, pad_right=True, left_pad_symbol='_', right_pad_symbol='_'))) 

cg_char_bigrams_flat_tuples = [bigram for bigram_list in cg_char_bigrams for bigram in bigram_list]
smg_char_bigrams_flat_tuples = [bigram for bigram_list in smg_char_bigrams for bigram in bigram_list]

cg_char_bigrams_flat = ['%s%s' % bigram_tuple for bigram_tuple in cg_char_bigrams_flat_tuples]
smg_char_bigrams_flat = ['%s%s' % bigram_tuple for bigram_tuple in smg_char_bigrams_flat_tuples]

cg_char_bigrams_Text = Text(cg_char_bigrams_flat)
smg_char_bigrams_Text = Text(smg_char_bigrams_flat)
cg_char_bigrams_Text<jupyter_output><empty_output><jupyter_text>3.2.4.2 Trigrams `Text` objects<jupyter_code>import warnings

cg_char_trigrams = []
smg_char_trigrams = []

with warnings.catch_warnings():
    warnings.filterwarnings('ignore',category=DeprecationWarning)
    for word in cg_words_flat:
        if len(word) > 1: # Trigram features for 1-letter words are useless and encoded by other features I use.
            cg_char_trigrams.append(list(ngrams(word, 3, pad_left=True, pad_right=True, left_pad_symbol='_', right_pad_symbol='_')))
    for word in smg_words_flat:
        if len(word) > 1:
            smg_char_trigrams.append(list(ngrams(word, 3, pad_left=True, pad_right=True, left_pad_symbol='_', right_pad_symbol='_')))

# Removing redundant trigrams:
cg_char_trigrams = [trigram_list[1:-1] for trigram_list in cg_char_trigrams]
smg_char_trigrams = [trigram_list[1:-1] for trigram_list in smg_char_trigrams]

cg_char_trigrams_flat_tuples = [trigram for trigram_list in cg_char_trigrams for trigram in trigram_list]
smg_char_trigrams_flat_tuples = [trigram for trigram_list in smg_char_trigrams for trigram in trigram_list]

cg_char_trigrams_flat = ['%s%s%s' % trigram_tuple for trigram_tuple in cg_char_trigrams_flat_tuples]
smg_char_trigrams_flat = ['%s%s%s' % trigram_tuple for trigram_tuple in smg_char_trigrams_flat_tuples]

cg_char_trigrams_Text = Text(cg_char_trigrams_flat)
smg_char_trigrams_Text = Text(smg_char_trigrams_flat)
cg_char_trigrams_Text<jupyter_output><empty_output><jupyter_text>4. Analysis  4.1. Corpus size<jupyter_code>print('Number of CG sentences:', len(cg_sents_clean))
print('Number of SMG sentences:', len(smg_sents_clean))
print('Number of words in CG data:', len(cg_words_flat))
print('Number of words in SMG data:', len(smg_words_flat))<jupyter_output>Number of words in CG data: 7026
Number of words in SMG data: 7100<jupyter_text>4.2 Most frequent words and characters  4.2.1 Most frequent words  4.2.1.1 CG<jupyter_code>cg_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.2.1.2 SMG<jupyter_code>smg_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.2.2 Most frequent characters  4.2.2.1 CG<jupyter_code>cg_chars_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.2.2.2 SMG<jupyter_code>smg_chars_Text.plot(10)<jupyter_output><empty_output><jupyter_text>NB: Recall that 'σ' in the most frequent character charts above includes instances of both 'σ' and 'ς'.  4.3 Most frequent word n-grams  4.3.1 Bigrams  4.3.1.1 CG<jupyter_code>cg_word_bigrams_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.3.1.2 SMG<jupyter_code>smg_word_bigrams_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.4 Most frequent character n-grams  4.4.1 Bigrams  4.4.1.1 CG<jupyter_code>cg_char_bigrams_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.4.1.2 SMG<jupyter_code>smg_char_bigrams_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.4.2 Trigrams  4.4.2.1 CG<jupyter_code>cg_char_trigrams_Text.plot(10)<jupyter_output><empty_output><jupyter_text>4.4.2.2 SMG<jupyter_code>smg_char_trigrams_Text.plot(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15072394967956326, 'fraction_numerical': 0.01392515230635335, 'mean_word_length': 5.282306163021869, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 77, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 13, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '400403', 'license': 'MIT', 'n_tokens_mistral': 5480, 'n_tokens_neox': 5141, 'n_words': 1067, 'path': '2-Data-Analysis.ipynb', 'repo_name': 'hb20007/greek-dialect-classifier', 'size': '12639'}