<commit_before>__author__ = 'harsh'
import nltk
from itertools import tee, islice
from collections import Counter

def preprocess(filename):
    f = open(filename)
    html = f.read()
    raw = nltk.clean_html(html)
    words = raw.split()
    return list(words)

def ngram(lst, n):
    tlst = lst
    while True:
        a, b = tee(tlst)
        l = tuple(islice(a, n))
        if len(l) == n:
            yield l
            next(b)
            tlst = b
        else:
            break



def ngrams(filename, n):
    filename = 'test1.html'
    ngrams_dict = Counter(ngram(preprocess(filename), n))
    ngrams_len = sum(ngrams_dict.itervalues())

    ngrams_probability_dict = {}
    for key, value in ngrams_dict.iteritems():
        ngrams_probability_dict[key] = value/float(ngrams_len)

    return ngrams_probability_dict

def main():
    unigrams = ngrams('test1.html', 1)
    bigrams = ngrams('test1.html', 2)
    print "Unigrams .........."
    print unigrams
    print "Bigrams................"
    print bigrams
    return 1
<commit_msg>multiple edits<commit_after>__author__ = 'harsh'

import re
import sys
import random
import operator

from nltk import clean_html, tokenize, PunktWordTokenizer
from collections import Counter


def preprocess(file_contents, add_sent_markers=True):
    raw = clean_html(file_contents)
    raw = re.sub('\d:\d ', "", raw)
    sentence_list = tokenize.sent_tokenize(raw)
    if add_sent_markers:
        sentence_list = [('<s> ' + sentence + ' </s>') for sentence in sentence_list]
    word_lists = [PunktWordTokenizer().tokenize(sentence) for sentence in sentence_list]
    word_list = [item.lower() for sublist in word_lists for item in sublist]
    return word_list


def generate_bigrams_list(word_list):
    return zip(word_list, word_list[1:])


def uni_bi_grams(file_contents):
    word_list = preprocess(file_contents)
    unigrams_dict = Counter(word_list)
    bigrams_dict = Counter(generate_bigrams_list(word_list))
    unigrams_len = sum(unigrams_dict.itervalues())

    unigrams_probability_dict = {}
    for key, value in unigrams_dict.iteritems():
        unigrams_probability_dict[key] = round(value/float(unigrams_len), 6)

    bigrams_probability_dict = {}
    for key, value in bigrams_dict.iteritems():
        bigrams_probability_dict[key] = round(value/float(unigrams_dict[key[0]]), 4)

    return unigrams_probability_dict, bigrams_probability_dict


def unigram_random_sentence_generator(unigrams_prob_dict, sent_length=20):
    sentence = ''
    padding = 0.05
    max_unigrams_prob = max(unigrams_prob_dict.iteritems(), key=operator.itemgetter(1))
    while len(sentence.split()) < sent_length:
        rand_prob = random.uniform(0.0, max_unigrams_prob[1] + padding)
        closest_key, closest_val = min(unigrams_prob_dict.iteritems(), key=lambda (k, v): abs(v - rand_prob))
        if abs(rand_prob - closest_val) > 0.01:
            continue
        if closest_key == '</s>':
            break
        sentence = sentence + ' ' + closest_key

    return re.sub('<[^<]+?>', '', sentence)


def bigram_random_sentence_generator(bigrams_prob_dict, sent_length=30):
    sentence = ''
    mru_word = '<s>'
    while len(sentence.split()) < sent_length and '</s>' not in sentence:
        rand_prob = random.uniform(0.0, 1.0)
        short_bigram_dict = dict((k, v) for k, v in bigrams_prob_dict.iteritems() if k[0] == mru_word)
        closest_key, closest_val = min(short_bigram_dict.iteritems(), key=lambda (k, v): abs(v - rand_prob))
        if '<s>' not in sentence:
            if closest_key[0] == '<s>':
                sentence += closest_key[0] + " " + closest_key[1]
            else:
                continue
        else:
            mru_word = sentence.split()[-1]
            if closest_key[0] == mru_word:
                sentence += " " + closest_key[1]
            else:
                continue

    return re.sub('<[^<]+?>', "", sentence)


def main():
    if len(sys.argv) < 2:
        print "Please enter the name of a corpus file as a command line argument."
        sys.exit()

    try:
        file_obj = open(sys.argv[1])
    except IOError:
        print "ERR: %s not found in current directory", sys.argv[1]
        sys.exit()

    file_contents = file_obj.read()
    unigrams, bigrams = uni_bi_grams(file_contents)
    '''print "Unigrams .........."
    print unigrams
    print "Bigrams................"
    print bigrams'''
    print 'Unigram Random Sentence Generator...'
    unigrams_sent = unigram_random_sentence_generator(unigrams)
    print unigrams_sent
    print 'Bigram Random Sentence Generator...'
    bigrams_sent = bigram_random_sentence_generator(bigrams)
    print bigrams_sent
    return True

{'fraction_non_alphanumeric': 0.12186836518046709, 'fraction_numerical': 0.008067940552016985, 'mean_word_length': 2.906301824212272, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6302626', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1536, 'n_words': 350}