<commit_before>#!/usr/bin/python

"""Python script to generate a dictionary given a url and starting and ending delimeters

Example:
    HAWAIIAN DICTIONARY
    python gen-dictionary.py -u "http://hawaiian-words.com/hawaiian-dictionary"
    -o "hawaiian.dic" -s "<tr    ><td >" -e "</td>"
"""

import sys
import getopt
import urllib
from util import mquit


def main():
    """Main function to run the script"""
    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(argv, "u:o:s:e:")
    except getopt.GetoptError:
        mquit('gen-dictionary.py -u <URL> -o <outputfile> -s <starting delimeter> -e <ending delimeter>')

    #  All options must be specified
    if len(opts) < 4:
        mquit('gen-dictionary.py -u <URL> -o <outputfile> -s <starting delimeter> -e <ending delimeter>')

    url = ''
    output = ''
    start = ''
    end = ''
    #  get and set all the user defined options
    for opt, arg in opts:
        if opt == '-u':
            url = arg
        elif opt == '-o':
            output = arg
        elif opt == '-s':
            start = arg
        elif opt == '-e':
            end = arg

    #  require all options, if not exit
    if url == '' or output == '' or start == '' or end == '':
        mquit('gen-dictionary.py -u <URL> -o <outputfile> -s <starting delimeter> -e <ending delimeter>')

    print "Openning output file: " + output
    outputfile = open(output, "w")
    print "Connecting to: " + url
    site = urllib.urlopen(url)

    #  Begin parsing
    #  if the line contains the start delimeter and the end delimeter take what's between the delimeters
    print "Parsing..."
    line = site.readline()
    while line != "":
        if line.find(start) != -1 and line.find(end) != -1:
            outputfile.write(line[line.find(start) + len(start): line.find(end)] + "\n")<commit_msg>Fixed trailing whitespace bug in gen-dictionary.py<commit_after>#!/usr/bin/python

"""Python script to generate a dictionary given a url and starting and ending delimeters

Example:
    HAWAIIAN DICTIONARY
    python gen-dictionary.py -u "http://hawaiian-words.com/hawaiian-dictionary"
    -o "hawaiian.dic" -s "<tr    ><td >" -e "</td>"
"""

import sys
import getopt
import urllib
from util import mquit


def main():
    """Main function to run the script"""
    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(argv, "u:o:s:e:")
    except getopt.GetoptError:
        mquit('gen-dictionary.py -u <URL> -o <outputfile> -s <starting delimeter> -e <ending delimeter>')

    #  All options must be specified
    if len(opts) < 4:
        mquit('gen-dictionary.py -u <URL> -o <outputfile> -s <starting delimeter> -e <ending delimeter>')

    url = ''
    output = ''
    start = ''
    end = ''
    #  get and set all the user defined options
    for opt, arg in opts:
        if opt == '-u':
            url = arg
        elif opt == '-o':
            output = arg
        elif opt == '-s':
            start = arg
        elif opt == '-e':
            end = arg

    #  require all options, if not exit
    if url == '' or output == '' or start == '' or end == '':
        mquit('gen-dictionary.py -u <URL> -o <outputfile> -s <starting delimeter> -e <ending delimeter>')

    print "Openning output file: " + output
    outputfile = open(output, "w")
    print "Connecting to: " + url
    site = urllib.urlopen(url)

    #  Begin parsing
    #  if the line contains the start delimeter and the end delimeter take what's between the delimeters
    print "Parsing..."
    line = site.readline()
    while line != "":
        if line.find(start) != -1 and line.find(end) != -1:
            outputfile.write(line[line.find(start) + len(start): line.find(end)].rstrip() + "\n")
{'fraction_non_alphanumeric': 0.14136266094420602, 'fraction_numerical': 0.002145922746781116, 'mean_word_length': 2.3564356435643563, 'pattern_counts': {'":': 2, '<': 35, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '801192', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1185, 'n_words': 396}