<jupyter_start><jupyter_text>Cleaning the newspaper archive<jupyter_code>import os
import re
import json
import itertools

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline

from functools import partial
from itertools import product
from collections import Counter
from unicodedata import category
from multiprocessing import cpu_count
from utils import multicore_apply

import fasttext
from umap import UMAP
from scipy.linalg import norm
from reo_toolkit import is_maori
from gensim.models import Phrases
from nltk.tokenize import word_tokenize, sent_tokenize

import warnings
from numba.errors import NumbaWarning
warnings.filterwarnings("ignore", category = UserWarning)
warnings.simplefilter("ignore", category = NumbaWarning)
def remove_punctuation(text):
    return ''.join(ch for ch in text if category(ch)[0] != 'P')
    
def normalize_text(text):
    text = re.sub("\s{2,}", " ", text)
    return remove_punctuation(text.lower())<jupyter_output><empty_output><jupyter_text>Cleaning the text<jupyter_code>with open('../data/output.json', 'r') as f:
    papers = pd.DataFrame(json.load(f))
col_patterns = {
    'newspaper_id': '([A-Z]+)[^/]+$',
    'year': '[A-Z]+([0-9]{4})[^/]+$',
    'month': '[A-Z]+[0-9]{4}([0-9]{2})[^/]+$',
    'day': '[A-Z]+[0-9]{6}([0-9]{2})[^/]+$',
    'id': '[A-Z]+[0-9]+\.([^/]+)$'
}

for col, pattern in col_patterns.items():
    papers[col] = papers.url.str.extract(pattern)

papers.head()
papers['paragraph'] = papers.text.apply(lambda s: s.split("\n"))
papers = papers.explode('paragraph')
%%time
papers['sentence'] = multicore_apply(papers.paragraph, sent_tokenize)
papers = papers.explode('sentence')
papers = papers[~papers.sentence.isna()]

papers.head()
# Print the number of sentences in te_ara
print("There are {} sentences in the māori newspaper archive".format(len(papers)))<jupyter_output>There are 805342 sentences in the māori newspaper archive<jupyter_text>Here are the first 5 sentences:<jupyter_code>print('\n'.join(papers.sentence[:5]))<jupyter_output>KO TE NIUPEPA MAORI.
EI te Turei tuatahi o te marama ka taia .
te taha maori o te Kai-whakamataara ngoa iwi erua ?
he mea huihui nga korero o nga wiki o te marama o mua.
Ko te utu mo te niupepa kotahi, etoru pene* *** Ko nga pukapuka katoa ki te Kai-wha-kamataara me kawe mai kite whare o np;a kai-ta pukapuka o Wiremuhana raua ko WiRIHANA.<jupyter_text>Text cleaning<jupyter_code>def extract_words(text):
    text = text.lower()
    results = []
    for word in word_tokenize(text):
        if re.search('[a-zāēīōū]', word):
            result = re.sub('\s{2,}', ' ',
                re.sub('[^a-zāēīōū]', ' ', word)
            )
            for res in result.split():
                results.append(res)
    return results
%%time
papers['words'] = multicore_apply(papers['sentence'], extract_words)
word_counts = papers['words'].explode().value_counts()
word_counts = word_counts.to_frame().reset_index()
word_counts.columns = ['word', 'count']
word_counts.to_csv('../data/papers_word_counts.csv', index = False)
word_counts
def drop_macrons(text):
    macron_dict = {
        'ā': 'a',
        'ē': 'e',
        'ī': 'i',
        'ō': 'o',
        'ū': 'u',
        'Ā': 'A',
        'Ē': 'E',
        'Ī': 'I',
        'Ō': 'O',
        'Ū': 'U',
    }
    
    for k, v in macron_dict.items():
        text = text.replace(k, v)
    
    return text<jupyter_output><empty_output><jupyter_text>Phrase modellingWe can use gensim to join common phrases in the text.<jupyter_code>def phrase_model(lines, min_count, threshold, phrase_length):
    
    for _ in range(phrase_length):
        sentence_stream = [doc.split(" ") for doc in lines]
        bigram = Phrases(sentence_stream, min_count=min_count, threshold=threshold)
        lines = [' '.join(bigram[line.split()]) for line in lines]
        
    return lines
maori_words  = set(word for word in papers.loc[papers.words.apply(len) > 0, 'words'].explode().unique() if is_maori(word))
non_maori_words  = set(word for word in papers.loc[papers.words.apply(len) > 0, 'words'].explode().unique() if not is_maori(word))
%%time
papers['discarded'] = phrase_model(
    (papers.words
        .apply(lambda x: ' '.join(
            [y for y in x if y in non_maori_words]
        ))),
    min_count = 30,
    threshold = 10,
    phrase_length = 5
)
(pd.Series([w for l in papers['discarded'] for w in l.split()])
    .value_counts(ascending = True)
    .tail(300)
    .plot(kind = 'barh', figsize = (10, 60))
)
%%time
papers['phrase'] = phrase_model(
    (papers.words
        .apply(lambda x: ' '.join(
            [y for y in x if y in maori_words]
        ))),
    min_count = 30,
    threshold = 10,
    phrase_length = 5
)
%%time
phrase_counts = papers.phrase.apply(str.split).explode().value_counts().to_frame().reset_index()
phrase_counts.columns = ['word', 'count']
phrase_counts<jupyter_output>CPU times: user 7.14 s, sys: 485 µs, total: 7.14 s
Wall time: 7.13 s<jupyter_text>Train fasttext model<jupyter_code>corpus_file = '../data/papers_corpus.txt'
with open(corpus_file, 'w') as f:
    for line in papers.loc[papers.phrase.str.len() != 0, 'phrase']:
        f.write(line + '\n')
%%time
model = fasttext.train_unsupervised(corpus_file, minCount = 30)
model.save_model("../data/fasttext.bin")
model_data = pd.DataFrame({'word': model.words})
model_data = model_data[model_data.word != '</s>']
model_data['word_vector'] = model_data.word.apply(model.get_word_vector)
model_data
model_data = model_data.merge(phrase_counts, how = 'left', on = ['word'])
word_vectors = np.vstack(model_data.word_vector.apply(lambda x: x / norm(x)))
similarity_matrix = np.dot(word_vectors, word_vectors.transpose())
model_data['neighbours'] = (model_data
    .word
    .apply(lambda x: [(round(score, 2), model.get_word_id(word)) for score, word in model.get_nearest_neighbors(x)]))
model_data[['word', 'count']].to_csv('../data/model_data.csv', index = False)
model_data
model_data['word_id'] = model_data.word.apply(model.get_word_id)<jupyter_output><empty_output><jupyter_text>Dimensionality reduction with UMAP<jupyter_code>def calculate_umap(params, umap_data, similarity_matrix):
    neighbors, dist = params

    reducer = UMAP(n_neighbors = neighbors, min_dist = dist)
    reduced = reducer.fit_transform(similarity_matrix)

    umap_data['n_neighbors'] = neighbors
    umap_data['min_dist'] = dist
    
    umap_data['x'] = reduced[:, 0]
    umap_data['y'] = reduced[:, 1]

    umap_data = umap_data[['n_neighbors', 'min_dist', 'word', 'count', 'x', 'y']]

    umap_data['rank'] = np.arange(len(umap_data))

    umap_data = umap_data[umap_data.word != '</s>']

    umap_data.columns = ['n_neighbors', 'min_dist', 'word', 'word_count', 'x_coord', 'y_coord', 'rank']

    return umap_data
%%time
n_neighbours = [2, 5, 10, 20]
min_dist = [0.6, 0.8, 0.95, 0.99]

umap_results = pd.concat(
    multicore_apply(
        product(n_neighbours, min_dist), 
        partial(calculate_umap, 
                umap_data = model_data,
                similarity_matrix = similarity_matrix))
)
g = (sns.FacetGrid(umap_results, col="n_neighbors", row="min_dist", sharex = False, sharey = False)
    .map(plt.scatter, "x_coord", "y_coord", edgecolor="w")
)
umap_results['x_coord'] = umap_results.x_coord
umap_results['y_coord'] = -umap_results.y_coord
(umap_results
     .loc[
        (umap_results.n_neighbors == 10) & \
        (umap_results.min_dist == 0.8) & \
        ((umap_results.word_count > 30) |
        (umap_results.word.str.contains("_"))),
        ['word', 'word_count', 'x_coord', 'y_coord', 'rank']
     ].reset_index(drop = True)
      .to_csv('../data/umap.csv', index = False)
)
! mv ../data/umap.csv ../D3/newspapers/umap.csv<jupyter_output><empty_output><jupyter_text>Topic model<jupyter_code>sentences = (papers
     .groupby(['newspaper_id', 'year', 'month', 'day', 'id', 'paragraph', 'sentence'])
     .agg({'phrase': lambda x: ' '.join(x).strip()})
     .reset_index()
)
sentences = sentences[sentences.phrase.str.len() > 0]
sentences
paragraphs = (papers
     .groupby(['newspaper_id', 'year', 'month', 'day', 'id', 'paragraph'])
     .agg({'phrase': lambda x: ' '.join(x).strip()})
     .reset_index()
)
paragraphs = paragraphs[paragraphs.phrase.str.len() > 0]
paragraphs
from scipy.linalg import norm
paragraphs['word_vector'] = paragraphs.phrase.apply(model.get_word_vector).apply(lambda x: x / norm(x))
paragraph_matrix = np.vstack(paragraphs.word_vector)
reducer = UMAP(n_components = 5, n_neighbors = 10, min_dist = 0.8)
reduced = reducer.fit_transform(paragraph_matrix)
reduced.shape
from sklearn.cluster import DBSCAN
clusters = DBSCAN(metric='cosine', eps=0.001, min_samples=50).fit_predict(reduced)

num_clusters = len(np.unique(clusters))
print("{} clusters identified".format(num_clusters))

paragraphs['cluster'] = clusters
cluster_data = paragraphs.groupby('cluster').agg({'phrase': lambda s: ' '.join(s).strip()}).reset_index()
cluster_data.phrase
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer()
X = tfidf.fit_transform(cluster_data.phrase)
feature_array = np.array(tfidf.get_feature_names())
tfidf_sorting = np.argsort(X.toarray()).flatten()[::-1]

n = 3
top_n = feature_array[tfidf_sorting][:n]
X = X.toarray()
feature_names = tfidf.get_feature_names()
np.sum(X > 0, axis = 1)
[m for m in dir(model) if not m.startswith('_')]
model.get_analogies('patuone', 'maori', 'kawana_kerei')
def get_analogies(termA, termB, termC, k=15):
    return (pd.DataFrame(
        model.get_analogies(termA, termB, termC, k=k), 
        columns = ['similarity', 'term'])
       .sort_values('similarity')
       .plot(kind = 'barh', x = 'term', y = 'similarity', title = '{} is to {} as {} is to:'.format(termA, termB, termC))
    )

get_analogies('moana', 'awa', 'maunga')
def get_similar_terms(term, k = 15):
    return (pd.DataFrame(
        model.get_nearest_neighbors(term, k=k), 
        columns = ['similarity', 'term'])
       .sort_values('similarity')
       .plot(kind = 'barh', x = 'term', y = 'similarity', title = 'Top {} similar terms for: {}'.format(k, term))
    )
    
get_similar_terms('tohunga')
get_similar_terms('ratana')
num_top_words = 5
top_words = []
for i in range(num_clusters):
    tfidf_counts = X[i,:]
    sort_indices = np.argsort(-tfidf_counts)
    top_words.append([feature_names[idx] for idx in sort_indices[:num_top_words]])
print('\n\n'.join([' '.join(s) for s in top_words]))<jupyter_output>te nga ki ka ana

maori aite rironi rirongo rirongii

maori kiwi ia iho uhi

ki mai rote roto ote

te mete iutere iore tinge

ana kia kaua ina rirooea

maori aii hie rirooea riroo

maori waka pakeha te ihe

ko te aeoha pono tika

ka te ki mai nga

ihe aae riroo rironi rirongo

maori ko hoa te ia

taranaki aae riroo rironi rirongo

waikato aae riroo rironi rirongo

utu mote ka atu pauna

mauri mahi uu ooa towu

aotearoa te paunui aoteaeoa tearoa

pukapuka hoko ukapuka kapuka whare

akarana rirooea riroo rironi rirongo

powhiri te wehi ana ki

upoko upoki ika takiwa mau

te nui_tireni ko karere nga

panuitanga uitanga anuitanga aae rirooea

huia kotahi tangata aitua mano

tukua reta mai tekua ke

tangata aae rirongo rirongii rirongi

heretaunga aae rirooea riroo rironi

nama aae riroo rironi rirongo

rongo korero nga taina kingi

tupapaku tanumanga nehunga aae rironi

maehe tokio ra riroraua rirooea

panui aae riroo rironi rirongo

pitopito_korero pitopito nga te horero

whanganui aae ri[...]
{'fraction_non_alphanumeric': 0.1459257326724063, 'fraction_numerical': 0.00956604922167145, 'mean_word_length': 4.208333333333333, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '556463', 'license': 'MIT', 'n_tokens_mistral': 4476, 'n_tokens_neox': 4302, 'n_words': 982, 'path': 'analysis/cleaning_papers_past.ipynb', 'repo_name': 'dragonfly-science/papers-past', 'size': '11499'}