<jupyter_start><jupyter_text>Extract Data from Every Text File and Format in a Single CSV File<jupyter_code>allCallReasons = []

basePath = "./Text/Extracted"

outputPath = "./BrocktonPoliceRecords.txt"
out = open(outputPath, "w")

# write the first line which includes the column names
headerLine = "Date|Call_Number|Reference_Call_Number|Call_Time|Action|Call_Reason|Call_Taker|Location|Num_Officers|Officers|Num_Arrests|Num_Arrest_Charges|Arrests_Raw_Data|Num_Summons|Num_Summons_Charges|Summons_Raw_Data|Fire_Unit_Data|EMS_Unit_Data"
out.write(headerLine)
out.write("\n")

fileLocation = basePath

datesSorted = []

for filename in os.listdir(fileLocation):
    if filename.endswith(".txt"):
        path = fileLocation + '/' +filename
        datesSorted.append(path)

x.split('/')[-1].strip('.txt').split('.')[2]
        
datesSorted = sorted(datesSorted, key=lambda x: int(x.split('/')[-1].strip('.txt').split('.')[2] +
                                                    x.split('/')[-1].strip('.txt').split('.')[0] +
                                                    x.split('/')[-1].strip('.txt').split('.')[1]))

for path in datesSorted:
    filePath = path
    filename = path.split('/')[-1]
    f = open(filePath, "r")
    
    currentDate = filename.replace(".txt", "").replace(".", "/")
    callData = []
    for line in f:
        pieces = line.split(" ")
        # search for the pattern nn- for example 15-12466 or 18-120224
        patternFound = re.search("\d{2}-", pieces[0])
        # if we've found the beginning of a call
        if patternFound:
            # if this is not the first call
            if len(callData) != 0:
                # write to the file
                columns = extractData(callData)
                if len(columns) > 0:
                    # Add the date as the first column
                    columns = currentDate+"|"+columns
                    out.write(columns)
                    out.write("\n")
                callData = []
                callData.append(line)
            else:
                callData.append(line)
        else:
            # keep the unmodified line
            callData.append(line)

    # write to the file
    columns = extractData(callData)
    if len(columns) > 0:
        # Add the date as the first column
        columns = currentDate+"|"+columns
        out.write(columns)
        out.write("\n")
    callData = []
    f.close()

out.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1351461038961039, 'fraction_numerical': 0.01176948051948052, 'mean_word_length': 2.152173913043478, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '480039', 'license': 'Unlicense', 'n_tokens_mistral': 755, 'n_tokens_neox': 726, 'n_words': 161, 'path': 'Text to CSV.ipynb', 'repo_name': 'Vinnie-Singleton/Brockton_911_Data_Analysis', 'size': '2464'}