<commit_before>#!/usr/bin/env python3

# This script uses Hunspell to process arbitrary text strings and provide
# suggestions for spelling correction.
#
# Copyright (c) 2017 <NAME> <<EMAIL>>
#                    <NAME>  <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

import sys
import getopt
import os
import io
import json
import re
import hunspell


# this function prints usage information
def print_usage():
    print( 'Usage:')
    print( 'spell_check.py [--path=PATH] [--lang=LANG] [--correct]')
    print( '               [--outfile=OUT.json] [--english OR --quiet]')
    print( '               --infile=INPUTFILE')
    print()
    print( 'spell_check.py [-p PATH] [-l LANG] [-c] [-o OUT.json] [-q]')
    print( '               -i INPUTFILE')

    return


# this function returns true if a string is a valid complex, real, or integer
# number
def isnumber(string):
    try:
        complex(string)  # try casting the string as a complex type
    except ValueError:
        return False

    return True


# spcheck(string to_check, string lang, hunspell.HunSpell hun)
# returns two lists -- mispelled is a complicated structure that is handled in
# build_json() and print_data()
def spcheck(to_check, lang, hun):
    misspelled = []  # each element is (line_num, word_num, (word, suggest[]))
    correct_words = []  # each element is (line_num, word_num, word)
    json_string = ''


    #
    #    *_num is the global position ( line, word, char )
    #    *_pos relative position on the line ( word, char )
    #

    char_num = 0
    line_num = 0
    word_num = 0
    # use built in word matching pattern
    regex = re.compile(r'(\w+)')

    for line in to_check:
        char_pos= 0
        word_pos = 0
        for chunk in line.split():  # tokenize by whitespace delimeters?
            # simple regex to ensure that punctuation at the end of a word is
            # not passed to hunspell i.e. prevents false alarms
            res = regex.match(chunk)
            if res:
                word = res.group(0)

                # note that hunspell will think that cat!dog is not a word, but
                # hyphenated words such as cat-dog may pass, even though they
                # aren't in the English lexicon

                char_pos = line.find(word, char_pos)

                if not hun.spell(word):
                    sug = [ x.decode('utf-8') for x in hun.suggest(word) ]
                    suggestion = (line_num, word_num, char_num+char_pos,
                                  char_pos, word_pos, (word, sug))
                    misspelled.append(suggestion)

                else:
                    correct_words.append((line_num, word_num, word))

                word_num += 1
                word_pos += 1

        line_num +=1
        char_num += len(line)

    feedback_stats = []
    # Note Key value shouldn't be changed unless changing in IFS
    # Storing Counts for correct and incorrect
    feedback_stats.append( { 'displayName': "Correct Word Count", 'key': 'correctWordCount', 'score': len(correct_words), 'level': 'basic' } )
    feedback_stats.append( { 'displayName': "Misspelled Word Count", 'key': 'misspelledWordCount', 'score': len(misspelled), 'level': 'basic' } )

    return misspelled, correct_words, feedback_stats

# build_feedbackStats_json(string filename, array of objects results)
# Results contains multiple items with key,level
# These attributes align with IFS Feedback_Mysql table
def build_feedbackStats_json(filename, results):
    shortFile = os.path.basename(filename)
    json_string = '"feedbackStats": [\n'
    for value in results:
        json_string += "{"
        json_string += '"type": "stat",\n'
        json_string += '"toolName": "hunspell",\n'
        json_string += '"name": "' + value['key'] + '",\n'
        json_string += '"level": "' + value['level'] + '",\n'
        json_string += '"category": "spelling",\n'
        json_string += '"filename": "' + shortFile + '",\n'
        json_string += '"statName": "' + value['displayName'] + '",\n'
        json_string += '"statValue": "' + str(value['score']) + '"\n'<commit_msg>cleaned up spell_check.py code a little bit<commit_after>#!/usr/bin/env python3

# This script uses Hunspell to process arbitrary text strings and provide
# suggestions for spelling correction.
#
# Copyright (c) 2017 <NAME> <<EMAIL>>
#                    <NAME>  <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
# REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
# OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
# PERFORMANCE OF THIS SOFTWARE.

import sys
import getopt
import os
import io
import json
import re
import hunspell


# this function prints usage information
def print_usage():
    print( 'Usage:')
    print( 'spell_check.py [--path=PATH] [--lang=LANG] [--correct]')
    print( '               [--outfile=OUT.json] [--english OR --quiet]')
    print( '               --infile=INPUTFILE')
    print()
    print( 'spell_check.py [-p PATH] [-l LANG] [-c] [-o OUT.json] [-q]')
    print( '               -i INPUTFILE')

    return


# this function returns true if a string is a valid complex, real, or integer
# number
def isnumber(string):
    try:
        complex(string)  # try casting the string as a complex type
    except ValueError:
        return False

    return True


# spcheck(string to_check, string lang, hunspell.HunSpell hun)
# returns two lists -- mispelled is a complicated structure that is handled in
# build_json() and print_data()
def spcheck(to_check, lang, hun):
    misspelled = []  # each element is (line_num, word_num, (word, suggest[]))
    correct_words = []  # each element is (line_num, word_num, word)
    json_string = ''

    #    *_num is the global position ( line, word, char )
    #    *_pos relative position on the line ( word, char )
    char_num = 0
    line_num = 0
    word_num = 0
    # use built in word matching pattern
    regex = re.compile(r'(\w+)')

    for line in to_check:
        char_pos= 0
        word_pos = 0
        for chunk in line.split():
            # simple regex to ensure that punctuation at the end of a word is
            # not passed to hunspell i.e. prevents false alarms
            res = regex.match(chunk)
            if res:
                word = res.group(0)
                char_pos = line.find(word, char_pos)

                if not hun.spell(word):
                    sug = [ x.decode('utf-8') for x in hun.suggest(word) ]
                    suggestion = (line_num, word_num, char_num+char_pos,
                                  char_pos, word_pos, (word, sug))
                    misspelled.append(suggestion)

                else:
                    correct_words.append((line_num, word_num, word))

                word_num += 1
                word_pos += 1

        line_num +=1
        char_num += len(line)

    feedback_stats = []
    # Note 'key' value shouldn't be changed unless changing in IFS
    # Storing Counts for correct and incorrect
    feedback_stats.append({
        'displayName': "Correct Word Count",
        'key': 'correctWordCount',
        'level': 'basic',
        'score': len(correct_words),
    })
    feedback_stats.append({
        'displayName': "Misspelled Word Count",
        'key': 'misspelledWordCount',
        'level': 'basic',
        'score': len(misspelled),
    })

    return misspelled, correct_words, feedback_stats


# build_feedbackStats_json(string filename, array of objects results)
# Results contains multiple items with key,level
# These attributes align with IFS Feedback_Mysql table
def build_feedbackStats_json(filename, results):
    shortFile = os.path.basename(filename)
    json_string = '"feedbackStats": [\n'
    for value in results:
        json_string += "{"
        json_string += '"type": "stat",\n'
        json_string += '"toolName": "hunspell",\n'
        json_string += '"name": "' + value['key'] + '",\n'
        json_string += '"level": "' + value['level'] + '",\n'
        json_string += '"category": "spelling",\n'
        json_string += '"filename": "' + shortFile + '",\n'
        json_string += '"statName": "' + value['displayName'] + '",\n'
        json_string += '"statValue": "' + str(value['score']) + '"\n'
{'fraction_non_alphanumeric': 0.10958178930651138, 'fraction_numerical': 0.0031762837480148226, 'mean_word_length': 2.5943683409436833, 'pattern_counts': {'":': 18, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5825093', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2681, 'n_words': 966}