<commit_before>import csv
import json

letterArray = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
data = {}

nGrams = ["1gram", "2gram", "3gram", "4gram", "5gram"]

def main():
    fileInput = '../data/baby-names/yob2016.txt'
    fileOutput = '../data/2016-5gram-b.json'

    # open csv file
    with open(fileInput, newline='') as f:
        print("Processing: " + fileInput)
        reader = csv.reader(f)

        # initialize frequency list for the length of letters list
        for gender in ["male", "female"]:
            data[gender] = {}
            for distribution in nGrams:
                data[gender][distribution] = {}
                data[gender][distribution]["frequency"] = {}

        # add counts for each letter
        for row in reader:
            name = row[0]
            if (row[1] == "M"):
                gender = "male"
            else:
                gender = "female"
            count = row[2]

            # iterate through letters and multiplies by the name count
            for i in range(len(name)):
                letterValue = name[i].lower()
                for distribution in nGrams:
                    letterKey = ""
                    if (distribution == "1gram"):
                        letterKey = "_"

                    elif (distribution == "2gram" and i >= 0):
                        if (i == 0):
                            letterKey = "_"
                        else:
                            letterKey = name[i - 1]

                    elif (distribution == "3gram" and i >= 1):
                        if (i == 1):
                            letterKey = "_" + name[i - 1]
                        else:
                            letterKey = name[i - 2] + name[i - 1]

                    elif (distribution == "4gram" and i >= 2):
                        if (i == 2):
                            letterKey = "_" + name[i - 2] + name[i - 1]
                        else:
                            letterKey = name[i - 3] + name[i - 2] + name[i - 1]

                    elif (distribution == "5gram" and i >= 3):
                        if (i == 3):
                            letterKey = "_" + name[i - 3] + name[i - 2] + name[i - 1]
                        else:
                            letterKey = name[i - 4] + name[i - 3] + name[i - 2] + name[i - 1]

                    if (letterKey != ""):
                        letterKey = letterKey.lower()
                        if (letterKey in data[gender][distribution]["frequency"]):
                            if (letterValue in data[gender][distribution]["frequency"][letterKey]):
                                data[gender][distribution]["frequency"][letterKey][letterValue] += int(count)
                            else:
                                data[gender][distribution]["frequency"][letterKey][letterValue] = int(count)
                        else:
                            data[gender][distribution]["frequency"][letterKey] = {}
                            data[gender][distribution]["frequency"][letterKey][letterValue] = int(count)

        # divide counts by total sum to get frequency
        '''
        for gender in ["male", "female"]:
            for distribution in nGrams:
                countSum = sum(data[gender][distribution]["frequency"].values())
                for letter, count in data[gender][distribution]["frequency"].items():
                    if (count > 0):
                        frequency = round(count / countSum * 100, 3)
                    else:
                        frequency = 0
                    data[gender][distribution]["frequency"][letter] = frequency
        '''
    # print json data
    #print(json.dumps(data, sort_keys=True, indent=2))

    # write json data
    with open(fileOutput, 'w') as f:
        print("Writing: " + fileOutput)
        json.dump(data, f, sort_keys=True)

if __name__ == "__main__":
    # execute only if run as a script
    main()<commit_msg>alt file processing cleanup<commit_after>import csv
import json

letterArray = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
ngramArray = ["1gram", "2gram", "3gram", "4gram", "5gram"]
genderArray = ["male", "female"]

data = {}

def main():
    fileInput = '../data/baby-names/yob2016.txt'
    fileOutput = '../data/2016-5gram-b.json'

    # open csv file
    with open(fileInput, newline='') as f:
        print("Processing: " + fileInput)
        reader = csv.reader(f)

        # initialize frequency list for the length of letters list
        for gender in genderArray:
            data[gender] = {}
            for ngram in ngramArray:
                data[gender][ngram] = {}

        # add counts for each letter
        for row in reader:
            name = row[0]
            if (row[1] == "M"):
                gender = "male"
            else:
                gender = "female"
            count = row[2]

            # iterate through letters and multiplies by the name count
            for i in range(len(name)):
                letterValue = name[i].lower()
                for ngram in ngramArray:
                    letterKey = ""
                    if (ngram == "1gram"):
                        letterKey = "_"

                    elif (ngram == "2gram" and i >= 0):
                        if (i == 0):
                            letterKey = "_"
                        else:
                            letterKey = name[i - 1]

                    elif (ngram == "3gram" and i >= 1):
                        if (i == 1):
                            letterKey = "_" + name[i - 1]
                        else:
                            letterKey = name[i - 2] + name[i - 1]

                    elif (ngram == "4gram" and i >= 2):
                        if (i == 2):
                            letterKey = "_" + name[i - 2] + name[i - 1]
                        else:
                            letterKey = name[i - 3] + name[i - 2] + name[i - 1]

                    elif (ngram == "5gram" and i >= 3):
                        if (i == 3):
                            letterKey = "_" + name[i - 3] + name[i - 2] + name[i - 1]
                        else:
                            letterKey = name[i - 4] + name[i - 3] + name[i - 2] + name[i - 1]

                    if (letterKey != ""):
                        letterKey = letterKey.lower()
                        if (letterKey in data[gender][ngram]):
                            if (letterValue in data[gender][ngram][letterKey]):
                                data[gender][ngram][letterKey][letterValue] += int(count)
                            else:
                                data[gender][ngram][letterKey][letterValue] = int(count)
                        else:
                            data[gender][ngram][letterKey] = {}
                            data[gender][ngram][letterKey][letterValue] = int(count)

        # divide counts by total sum to get frequency
        for gender in genderArray:
            for ngram in ngramArray:
                for letterKey in data[gender][ngram]:
                    letterCountSum = sum(data[gender][ngram][letterKey].values())
                    for letterValue, letterCount in data[gender][ngram][letterKey].items():
                        if (letterCount > 0):
                            letterFrequency = round(letterCount / letterCountSum * 100, 3)
                        else:
                            letterFrequency = 0
                        data[gender][ngram][letterKey][letterValue] = letterFrequency

    # print json data
    #print(json.dumps(data, sort_keys=True, indent=2))

    # write json data
    with open(fileOutput, 'w') as f:
        print("Writing: " + fileOutput)
        json.dump(data, f, sort_keys=True)

if __name__ == "__main__":
    # execute only if run as a script
    main()
{'fraction_non_alphanumeric': 0.1346321968277757, 'fraction_numerical': 0.013238416385662545, 'mean_word_length': 1.268555240793201, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4306820', 'n_tokens_mistral': 2314, 'n_tokens_neox': 2112, 'n_words': 586}