<jupyter_start><jupyter_text>Dataset gathering from wikimedia dumps---Here we gather dataset from the given link and unzip the folder and extract the files. I am using **'wikiextractor'** library to do the necessary commands.<jupyter_code>!wget http://dumps.wikimedia.org/tawiki/latest/tawiki-latest-pages-articles.xml.bz2
!bunzip2 tawiki-latest-pages-articles.xml.bz2
!ls -ltr
!git clone https://github.com/attardi/wikiextractor.git
!ls
!python ./wikiextractor/wikiextractor/WikiExtractor.py tawiki-latest-pages-articles.xml --no-templates -q<jupyter_output><empty_output><jupyter_text>Getting List of article collection files---<jupyter_code>import glob
flist=glob.glob('text/*/*')
len(flist)
flist[:2]
def future_name(fn):
  a,b,c=fn.split('/')
  return '/'.join([a,b,b+'_'+c+'.txt'])
future_name(flist[0])
import os
for f in flist:
  os.rename(f,future_name(f))
flist2=glob.glob('text/*/*')
len(flist2),flist2[:2]
with open(flist2[0], encoding='utf-8') as f:
    text=f.read()
print(text[:1000])<jupyter_output><doc id="3" url="https://ta.wikipedia.org/wiki?curid=3" title="முதற் பக்கம்">
முதற் பக்கம்

&lt;templatestyles src="Main Page/minerva.css" /&gt;

</doc>
<doc id="12" url="https://ta.wikipedia.org/wiki?curid=12" title="கட்டிடக்கலை">
கட்டிடக்கலை

கட்டிடக்கலை என்பது கட்டிடங்கள் மற்றும் அதன் உடல் கட்டமைப்புகளை வடிவமைத்தல், செயல்முறைத் திட்டமிடல், மற்றும் கட்டிடங்கள் கட்டுவதை உள்ளடக்கியதாகும். கட்டடக்கலை படைப்புகள், கட்டிடங்கள் பொருள் வடிவம், பெரும்பாலும் கலாச்சார சின்னங்களாக மற்றும் கலை படைப்புகளாக காணப்படுகின்றது. வரலாற்று நாகரிகங்கள் பெரும்பாலும் அவர்களின் கட்டிடகலை சாதனைகளின் மூலம் அடையாளம் காணப்படுகின்றன.
ஒரு விரிவான வரைவிலக்கணம், பெருமட்டத்தில், நகரத் திட்டமிடல், நகர்ப்புற வடிவமைப்பு மற்றும் நிலத்தோற்றம் முதலியவற்றையும், நுண்மட்டத்தில், தளபாடங்கள், உற்பத்திப்பொருள் முதலியவற்றை உள்ளடக்கிய, முழு உருவாக்கச் சூழலின் வடிவமைப்பைக் கட்டிடக்கலைக்குள் அடக்கும். 
மேற்படி விடயத்தில், தற்போது கிடைக்கும் மிகப் பழைய ஆக்கம், கி.பி. முதலாம் நூற்றாண்டைச் சேர்ந்த உரோமானியக் கட்டடக் கலைஞரான விட்ருவியஸ் [...]<jupyter_text>Extract titles of articles<jupyter_code>import re

example_title ='<doc id="12" url="https://ta.wikipedia.org/wiki?curid=12" title="கட்டிடக்கலை">'
pattern = 'title="(.*?)">'
with open(flist2[0], encoding='utf-8') as f:
    text=f.read()
    titles=re.findall(pattern, text)
    print(len(titles), 'articles found')<jupyter_output>84 articles found<jupyter_text>Total number of all articles<jupyter_code>pattern = 'title="(.*?)">'
def get_article_count(fname):
    with open(fname, encoding='utf-8') as f:
        text=f.read()
        titles=re.findall(pattern, text)
        return len(titles)
print(sum([get_article_count(f) for f in flist2]))<jupyter_output>185250<jupyter_text>Create tiny subset for analyses<jupyter_code>!mv text tawiki_large
flist3 = glob.glob('tawiki_large/*/*')
flist3[0], len(flist3)
import random
random.shuffle(flist3)
flist3[:2]
flist_small = flist3[:40]
!mkdir tawiki_small
for file in flist_small:
    with open(file, encoding='utf-8') as f:
        text = f.read()
        name=file.split('/')
        newname=name[0].replace('large', 'small')+'/'+name[2]
    with open(newname, "w") as text_file:
        text_file.write(text)

pattern = 'title="(.*?)">'
def get_article_count(fname):
    with open(fname, encoding='utf-8') as f:
        text=f.read()
        titles=re.findall(pattern, text)
        return len(titles)
    
flist_small = glob.glob('tawiki_small/*')
print(sum([get_article_count(f) for f in flist_small]))<jupyter_output>13597<jupyter_text>Dataset Preprocessing and Tokenization of tamil words for selected text file---<jupyter_code>import nltk, re, string, collections
from nltk.util import ngrams # function for making ngrams

# this corpus is pretty big, so let's look at just one of the files in it
with open("/content/drive/MyDrive/nlp_da1/tawiki_large/AA/AA_wiki_00.txt", "r") as file:
    text = file.read()

# check to make sure the file read in alright; let's print out the first 1000 characters
text[0:1000]

# get rid of all the XML markup
text = re.sub('<.*>','',text)

# get rid of the "ENDOFARTICLE." text
text = re.sub('ENDOFARTICLE.','',text)

text = re.sub('[a-zA-Z]','',text)

# get rid of punctuation
punctuationNoPeriod = "[" + re.sub("\.","",string.punctuation) + "]"
text = re.sub(punctuationNoPeriod, "", text)

# make sure it looks ok
text[0:1000]<jupyter_output><empty_output><jupyter_text>N-gram analyses (2,3,4)<jupyter_code># first get individual words
tokenized = text.split()

# and get a list of all the bi-grams
esBigrams = ngrams(tokenized, 2)

# and get a list of all the tri-grams
esBigrams2 = ngrams(tokenized, 3)

# and get a list of all the quad-grams
esBigrams3 = ngrams(tokenized, 4)

# If you like, you can uncomment the next like to take a look at 
# the first ten to make sure they look ok. Please note that doing so 
# will consume the generator & will break the next block of code, so you'll
# need to re-comment it and run this block again to get it to work.
#list(esBigrams)[:10]
# get the frequency of each bigram in our corpus
esBigramFreq = collections.Counter(esBigrams)

# what are the ten most popular ngrams in this Spanish corpus?
esBigramFreq.most_common(20)
tokenized
import pandas as pd
bigrams_series = (pd.Series(nltk.ngrams(tokenized, 2)).value_counts())[:12]
bigrams_series<jupyter_output><empty_output><jupyter_text>Plotting of 20 most frequently occuring bigrams<jupyter_code>import matplotlib.pyplot as plt
from matplotlib import rc
import matplotlib as mlp
from pathlib import Path
import matplotlib.font_manager as fontmanager
nirm = Path('/content/drive/MyDrive/nlp_da1/vijaya.ttf')
tam_font = fontmanager.FontProperties(fname=nirm)
bigrams_series.sort_values().plot.barh(color='blue', width=.9, figsize=(12, 8))
plt.title('20 Most Frequently Occuring Bigrams')
plt.yticks(fontproperties=tam_font)
plt.ylabel('Bigram')
plt.xlabel(' # of Occurances')<jupyter_output><empty_output><jupyter_text>plotting of most frequently occuring unigrams<jupyter_code>from collections import Counter
import seaborn as sns
import matplotlib.font_manager as fontmanager
import matplotlib as mpl
import matplotlib.pyplot as plt
frequency = Counter(tokenized)
df = pd.DataFrame(frequency.most_common(30))
plt.rcParams['figure.figsize'] = [12, 15]
df.columns =['Word', 'Frequency']
df_sorted= df.sort_values('Frequency')
df_sorted.head()
sns.set(font_scale = 1.3, style = 'whitegrid')
nirm = Path('/content/drive/MyDrive/nlp_da1/vijaya.ttf')
tam_font = fontmanager.FontProperties(fname=nirm)

# plotting
fig = plt.figure(figsize=(30, 25))
ax = df_sorted.plot.barh(x='Word', y='Frequency')
for i in ax.patches:
    plt.text(i.get_width()+0.2, i.get_y()+0.5,
             str(round((i.get_width()), 2)),
             fontsize=10, fontweight='bold',
             color='grey')
plt.title('Word Count')
plt.yticks(fontproperties=tam_font)
plt.ylabel('word')
plt.xlabel(' # of Occurances')<jupyter_output><empty_output><jupyter_text>Compiling all text files together for complete analysesHere we will compile the text files, do tokenization process then save all texts files together in one file---<jupyter_code>import os
for dirname, _, filenames in os.walk('/content/drive/MyDrive/nlp_da1/tawiki_small/'):
    for filename in filenames:
        with open('corpus.txt', 'a', encoding='latin-1') as ffile:
            with open(os.path.join(dirname, filename), 'r', encoding='latin-1') as rfile:
                ffile.write(rfile.read())
def getListWordsPreprocessed(corpus):
    ''' return a corpus after removing all the patterns and the xml markup (if any) '''
    text = corpus.lower()
    text = re.sub('<.*>', '', text)
    text = re.sub('ENDOFARTICLE.', '', text)
    punctuation2remove = "[" + re.sub('[,.;:?!()+/-]', '', string.punctuation) + "]"
    text = re.sub(punctuation2remove, '', text)
    text = re.sub('\n\n+', '\n', text)
    text = re.sub(';+\n', '\n', text)
    text = re.sub('\s*-\s', ' ', text)
    text = re.sub('\s+\.', ' ', text)
    text = re.sub('^\n', '', text, flags=re.MULTILINE)
    text = re.sub('^\s*\w+\s*\n', '', text, flags=re.MULTILINE)
    text = re.sub('\((\s*|\+*|\w\.\s*)\d+(\-*|\s*|,\s*)\d*\-*\)', ' ', text)
    text = re.sub('\(\s*\)', ' ', text)
    text = text.replace(',', '')
    text = text.replace('.', '')
    text = text.replace(';', '')
    text = text.replace(':', '')
    text = text.replace('?', '')
    text = text.replace('!', '')
    text = text.replace('(', '')
    text = text.replace(')', '')
    text = text.replace('/', '')
    text = text.replace('+', '')
    text = text.replace('-', '')
    text = re.sub('\s\d+\s', '', text)
    words = text.split()
    #remove all words with 5 or fewer occurences
    word_cnts = Counter(words)
    trimmed_words = [word for word in words if word_cnts[word] > 5]
    return trimmed_words<jupyter_output><empty_output><jupyter_text>Frequently used tamil wordHere we do process to find the most frequently used tamil words by referring the given dataset<jupyter_code>with open('./corpus.txt', 'r') as f:
    text = f.read()
words = getListWordsPreprocessed(text)
print(words[:50])
print("Total amount of words: {}".format(len(words)))
print("Amount of unique words: {}".format(len(set(words))))
# creating a counter of words ...
vocabulary_counts = Counter(words)
# let's see the 10 most common words
print("10 most commmon words:")
print(vocabulary_counts.most_common(10))

# sorting the words in order of frequency (from most to least frequent)
vocabulary_sorted = sorted(vocabulary_counts, key=vocabulary_counts.get, reverse=True)

# creating the lookup tables
int_to_vocab = {ii: word for ii, word in enumerate(vocabulary_sorted)}
vocab_to_int = {word: ii for ii, word in int_to_vocab.items()}

# create a vocabulary of ints (i..e map the complete vocabulary to its int values)
int_vocabulary = [vocab_to_int[word] for word in words]
print("First 50 int-words of the int vocabulary:")
print(int_vocabulary[:50])
frequent= vocabulary_counts.most_common(10)
frequent
freq = pd.DataFrame(frequent)
freq<jupyter_output><empty_output><jupyter_text>Plotting 10 most freqeuntly used tamil words using barplot<jupyter_code>from collections import Counter
import seaborn as sns
import matplotlib.font_manager as fontmanager
import matplotlib as mpl
import matplotlib.pyplot as plt
frequency = Counter(tokenized)
df = pd.DataFrame(frequency.most_common(30))
plt.rcParams['figure.figsize'] = [12, 15]
freq.columns =['Word', 'Frequency']
freq_sorted= freq.sort_values('Frequency')
sns.set(font_scale = 1.3, style = 'whitegrid')
nirm = Path('/content/drive/MyDrive/nlp_da1/vijaya.ttf')
tam_font = fontmanager.FontProperties(fname=nirm)

# plotting
fig = plt.figure(figsize=(40, 35))
ax = freq_sorted.plot.barh(x='Word', y='Frequency')
for i in ax.patches:
    plt.text(i.get_width()+0.2, i.get_y()+0.5,
             str(round((i.get_width()), 2)),
             fontsize=10, fontweight='bold',
             color='grey')
plt.title('Word Count')
plt.yticks(fontproperties=tam_font)
plt.ylabel('word')
plt.xlabel(' # of Occurances')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17122483978698438, 'fraction_numerical': 0.01462225832656377, 'mean_word_length': 4.99242833964305, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 55, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '301218', 'license': 'MIT', 'n_tokens_mistral': 4705, 'n_tokens_neox': 4803, 'n_words': 961, 'path': 'corpora__analysis.ipynb', 'repo_name': 'ilakkiya-v/tamil-corpora-analysis', 'size': '11079'}