<commit_before>
import sys, getopt
from summarizer import summarize
from keywords import keywords

TEST_FILE = "samples/textrank_example.txt"
# Types of summarization
SENTENCE = 0
WORD = 1


def get_arguments():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "t:s:l:h", ["text=", "summary=", "length=", "help"])
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit(2)
    path = TEST_FILE
    summarize_by = SENTENCE
    length = 0.2
    for o, a in opts:
        if o in ("-t", "--text"):
            path = a
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("-s", "--summary"):
            summarize_by = int(a)
        elif o in ("-l", "--length"):
            length = float(a)
        else:
            assert False, "unhandled option"

    return path, summarize_by, length


help_text = """Usage: python summa.py
-s UNIT, --summary=UNIT:
\tType of unit to summarize: sentence or word. Default value: 0
\t0: Sentence. 1: Word
-t PATH, --text=PATH:
\tPATH to text to summarize. Default value: samples/textrank_example.txt
-l LENGTH, --length=LENGTH:
\tFloat number (0,1] that defines the length of the summary. It's a proportion of the original text. Default value: 0.2
-h, --help:
\tprints this help
"""
def usage():
    print help_text


def textrank(text, summarize_by=SENTENCE, summary_length=0.2):
    if summarize_by == SENTENCE:
        return summarize(text, summary_length)
    else:
        return keywords(text, summary_length)


def main():
    path, summarize_by, length = get_arguments()

    with open(path) as test_file:
        text = test_file.read()

    print textrank(text, summarize_by, length)


if __name__ == "__main__":
    main()<commit_msg>Adding word limit to options.<commit_after>
import sys, getopt
from summarizer import summarize
from keywords import keywords

TEST_FILE = "samples/textrank_example.txt"
# Types of summarization
SENTENCE = 0
WORD = 1


def get_arguments():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "t:s:r:w:h", ["text=", "summary=", "ratio=", "words=", "help"])
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit(2)
    path = TEST_FILE
    summarize_by = SENTENCE
    ratio = 0.2
    words = None
    for o, a in opts:
        if o in ("-t", "--text"):
            path = a
        elif o in ("-h", "--help"):
            usage()
            sys.exit()
        elif o in ("-s", "--summary"):
            summarize_by = int(a)
        elif o in ("-w", "--words"):
            words = int(a)
        elif o in ("-r", "--ratio"):
            ratio = float(a)
        else:
            assert False, "unhandled option"

    return path, summarize_by, ratio, words


help_text = """Usage: python summa.py
-s UNIT, --summary=UNIT:
\tType of unit to summarize: sentence or word. Default value: 0
\t0: Sentence. 1: Word
-t PATH, --text=PATH:
\tPATH to text to summarize. Default value: samples/textrank_example.txt
-r RATIO, --ratio=RATIO:
\tFloat number (0,1] that defines the length of the summary. It's a proportion of the original text. Default value: 0.2.
-w WORDS, --words=WORDS:
\tNumber to limit the length of the summary. The length option is ignored if the word limit is set.
-h, --help:
\tprints this help
"""
def usage():
    print help_text


def textrank(text, summarize_by=SENTENCE, ratio=0.2, words=None):
    if summarize_by == SENTENCE:
        return summarize(text, ratio, words)
    else:
        return keywords(text, ratio, words)


def main():
    path, summarize_by, ratio, words = get_arguments()

    with open(path) as test_file:
        text = test_file.read()

    print textrank(text, summarize_by, ratio, words)


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.12971262852623253, 'fraction_numerical': 0.007909306617453203, 'mean_word_length': 2.5860113421550093, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5728572', 'n_tokens_mistral': 1277, 'n_tokens_neox': 1229, 'n_words': 415}