<commit_before>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word prediction based on Quadgram\n",
    "This program reads the corpus line by line so it is slower than the program which reads the corpus\n",
    "in one go.This reads the corpus one line at a time loads it into the memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#%%timeit\n",
    "from nltk.util import ngrams\n",
    "from collections import defaultdict\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Do preprocessing:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenize the corpus data\n",
    "## Remove the punctuations and lowercase the tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/seeker/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:46: DeprecationWarning: generator 'ngrams' raised StopIteration\n",
      "/home/seeker/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:40: DeprecationWarning: generator 'ngrams' raised StopIteration\n"
     ]
    }
   ],
   "source": [
    "tri_dict = defaultdict(int)            #for keeping count of sentences of three words\n",
    "quad_dict = defaultdict(int)           #for keeping count of sentences of three words    \n",
    "w1 = ''    #for storing the 3rd last word to be used for next token set\n",
    "w2 = ''    #for storing the 2nd last word to be used for next token set\n",
    "w3 = ''    #for storing the last word to be used for next token set\n",
    "vocab_dict = defaultdict(int) #for storing the different words with their frequencies\n",
    "#word_len = 0\n",
    "\n",
    "with open('corpusfile.txt','r') as file:\n",
    "    for line in file:\n",
    "        #tokenise each line\n",
    "        token = nltk.word_tokenize(line)\n",
    "        #word_len = word_len + len(token)\n",
    "    \n",
    "    \n",
    "        i =0\n",
    "        #remove punctuations and make it lowercase\\n\",\n",
    "        for item in token:\n",
    "            token[i] = item.lower()\n",
    "            token[i] = token[i].replace('.','').replace(',','').replace(':','').replace(';','').replace('!','').replace('?','').replace('(','').replace(')','')\n",
    "            i +=1\n",
    "\n",
    "        if not token:\n",
    "            continue\n",<commit_msg>Word Prediction using Quadgrams memory efficient version<commit_after>{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word prediction based on Quadgram\n",
    "This program reads the corpus line by line so it is slower than the program which reads the corpus\n",
    "in one go.This reads the corpus one line at a time loads it into the memory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#%%timeit\n",
    "from nltk.util import ngrams\n",
    "from collections import defaultdict\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Do preprocessing:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenize the corpus data\n",
    "## Remove the punctuations and lowercase the tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/seeker/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:46: DeprecationWarning: generator 'ngrams' raised StopIteration\n",
      "/home/seeker/anaconda3/lib/python3.6/site-packages/ipykernel/__main__.py:40: DeprecationWarning: generator 'ngrams' raised StopIteration\n"
     ]
    }
   ],
   "source": [
    "tri_dict = defaultdict(int)            #for keeping count of sentences of three words\n",
    "quad_dict = defaultdict(int)           #for keeping count of sentences of three words    \n",
    "vocab_dict = defaultdict(int) #for storing the different words with their frequencies\n",
    "w1 = ''    #for storing the 3rd last word to be used for next token set\n",
    "w2 = ''    #for storing the 2nd last word to be used for next token set\n",
    "w3 = ''    #for storing the last word to be used for next token set\n",
    "#word_len = 0\n",
    "\n",
    "with open('corpusfile.txt','r') as file:\n",
    "    for line in file:\n",
    "        #tokenise each line\n",
    "        token = nltk.word_tokenize(line)\n",
    "        #word_len = word_len + len(token)\n",
    "    \n",
    "    \n",
    "        i =0\n",
    "        #remove punctuations and make it lowercase\\n\",\n",
    "        for item in token:\n",
    "            token[i] = item.lower()\n",
    "            token[i] = token[i].replace('.','').replace(',','').replace(':','').replace(';','').replace('!','').replace('?','').replace('(','').replace(')','')\n",
    "            i +=1\n",
    "\n",
    "        if not token:\n",
    "            continue\n",
{'fraction_non_alphanumeric': 0.17801047120418848, 'fraction_numerical': 0.00758259613648673, 'mean_word_length': 2.392529087568892, 'pattern_counts': {'":': 56, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2976627', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1892, 'n_words': 475}