<commit_before>"""
This program generates an abstract for a report given a training corpus
Usage: python generate_abstract.py <path_to_corpus>
"""
import argparse
from nltk.parse.generate import generate
from corpora.corpora_parsing import processCorpus
from nltk import word_tokenize

def scoreSentence(s):
    '''
    Scores a given genrated sentence based on certain criteria
    '''

    score = 0

    # Duplicate words? Subtract from it's score
    score += len(set(s)) - len(s)

    return score

def main():

    # Config
    MAX_NUM_SENTENCES = 1000

    # Argument Parsing
    parser = argparse.ArgumentParser(description="""
        Generates a report abstract given a path to a corpus of abstracts
    """)
    parser.add_argument('path_to_corpus', help='Path to corpus of abstacts')
    args = parser.parse_args()

    # Find the corpus and get the ngrams dictionary and cfg grammer from it
    ngrams, cfg_grammer = processCorpus(args.path_to_corpus, verbose=True)

    print('Grammer done. Making sentences.')

    # Print out max sentences generated from the cfg
    best_score = float('-inf')
    best_s = ''
    for s in generate(cfg_grammer, depth=3, n=MAX_NUM_SENTENCES):
        s_score = scoreSentence(s)
        if s_score > best_score and len(s) > 0:
            best_score = s_score
            best_s = s
    print(best_s)
    #print(best_score)

if __name__ == '__main__':
    main()
<commit_msg>Added method to print sentence.<commit_after>"""
This program generates an abstract for a report given a training corpus
Usage: python generate_abstract.py <path_to_corpus>
"""
import argparse
from nltk.parse.generate import generate
from corpora.corpora_parsing import processCorpus
from nltk import word_tokenize

def scoreSentence(s):
    '''
    Scores a given genrated sentence based on certain criteria
    '''

    score = 0

    # Duplicate words? Subtract from it's score
    score += len(set(s)) - len(s)

    return score

def outputSentence(s):
    '''
    '''
    output = ''
    if len(s) > 0:
        output += (s[0][0].upper() + s[0][1:])  # make first word capital
        for word in s[1:]:
            output += ' ' + word.lower()
        output += '.'
    return output

def main():

    # Config
    MAX_NUM_SENTENCES = 100000

    # Argument Parsing
    parser = argparse.ArgumentParser(description="""
        Generates a report abstract given a path to a corpus of abstracts
    """)
    parser.add_argument('path_to_corpus', help='Path to corpus of abstacts')
    args = parser.parse_args()

    # Find the corpus and get the ngrams dictionary and cfg grammer from it
    ngrams, cfg_grammer = processCorpus(args.path_to_corpus, verbose=True)

    print('Grammer done. Making sentences.')

    # Print out max sentences generated from the cfg
    best_score = float('-inf')
    best_s = ''
    for s in generate(cfg_grammer, depth=3, n=MAX_NUM_SENTENCES):
        s_score = scoreSentence(s)
        if s_score > best_score and len(s) > 0:
            best_score = s_score
            best_s = s
    print(outputSentence(best_s))
    #print(best_score)

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.0942975469894871, 'fraction_numerical': 0.007008601465434852, 'mean_word_length': 3.09387222946545, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6677459', 'n_tokens_mistral': 1016, 'n_tokens_neox': 962, 'n_words': 306}