<jupyter_start><jupyter_text>Dataframe Creation Apply OCR routine to create pandas dataframe to posterior analysis.  OCR With Tesseract OCR part done with Tesseract to recognize the characters in the pdf's.<jupyter_code># Import modules for OCR
try:
    import Image
except ImportError:
    from PIL import Image
import pytesseract

# Import os for getting image filenames in img directory
from os import listdir

# Import pandas
import pandas as pd

# Import datetime
import datetime

# Import sys
import sys

# Import progress bar 2
import progressbar
# The cutoff represents the file where a new format starts
cutoff = "14-01-07.jpg"
# Livestock lot types
livestock_types = ["HL", "HV", "VE", "VP", "ML", "MC", "TO", "BH", "BM"]
# Categories of iron for female/male
iron_categories = ["HEMBRA DE 1a", "HEMBRA DE 2a", "MACHO DE 1a", "MACHO DE 2a", "HEMBRA DE PRIMERA", "MACHO DE PRIMERA", "HEMBRA DE SEGUNDA", "MACHO DE SEGUNDA"]
# Digits array
digits = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
# Number of faulty lines
faulty_lines = 0
# Total lines
total_lines = 0
# Image filenames 
image_filenames = [f for f in listdir("img") if ".jpg" in f]
# Auction filenames list
auction_filenames = []

# Temp ary of auction filename
temp_ary = []

# Generate auction filenames list
for index, filename in enumerate(image_filenames):
    # Append filename to temp_ary
    temp_ary.append(filename)
    # If it is the end append temp_ary no matter what
    if index == len(image_filenames)-1:
        auction_filenames.append(temp_ary)
    else:
        # Select next filename root to check if it belongs to the same multipage file
        next_root_filename = "-".join(image_filenames[index+1].replace(".jpg", "").split("-")[0:3])
        # If the next filename is another page of the same file go on and keep appending to temp_ary
        if next_root_filename in filename:
            pass
        # Else apppend temp ary and empty it
        else:
            auction_filenames.append(temp_ary)
            temp_ary = []
# Check if it coincides with the number of pdf's records gotten from the auction webstie
if len(auction_filenames) == 458:
    print("Ente gut, alles gut!")
# Return if the line corresponds to the category of an iron. If so, return the name of the category
def is_iron_category(line):
    # Check if there is an iron category in the line
    has_iron_category = [category in line for category in iron_categories]
    # Business logic
    if True in has_iron_category:
        return True, iron_categories[has_iron_category.index(True)] 
    return False, None
# Return if the line corresponds to a data entry. If so, return the line 
def is_data_entry(line):
    # Number of digits in the line
    number_of_digits = sum([line.count(digit) for digit in digits])
    # Bussiness logic
    has_livestock_type = [livestock_type in line for livestock_type in livestock_types]
    if (number_of_digits > 10) and (True in has_livestock_type):
        # Return True, the data line, the livestock type of the data line 
        return True, line, livestock_types[has_livestock_type.index(True)]
    return False, None, None
# Correct common errors in data line that come out of OCR routine
def formatted_data_line(line, livestock_type, date, iron_category):
    # Make total_lines and faulty_lines accesible
    global total_lines
    global faulty_lines
    # Replace " ," by ",". Some numbers have this pattern and appear broken
    line = line.replace(" ,", ",")
    # Remove double spaces if any
    while "  " in line:
        line.replace("  ", " ")
    # Remove commas and dots from numbers
    line = line.replace(",", "").replace(".", "")
    # Check if age exists or not
    if line.index(livestock_type) == 0: # If age is not present
        age, numerical_data = None, line.replace("{0} ".format(livestock_type), "")
    else: # If age is present
        age, numerical_data = line.split(" {0} ".format(livestock_type))
    # Check that the correct number of columns is in place
    if len(numerical_data.split(" ")) != 6:
        # Add to the number of faulty lines encountered
        faulty_lines += 1
        # Print line
        print("### Error in line: ###")
        print(line)
        return None
    # Add to the number of total lines processed
    total_lines +=1
    try:
        output = (date, age, livestock_type, *map(int, numerical_data.split(" ")), iron_category)
        return output
    except:
        print("### Error: ###")
        print(line)
        sys.exit("### sys.exit ###")
# Method that pulls data from image and append it to dataframe. It returns the last iron brand category 
def append_image_to_dataframe(auction_filename, iron_category, date):
    # Make df accesible
    global df
    # Variables to hold temp dataframe information
    data = []
    columns = ["date", "age", "type", "quantity", "weight", "price_min", "price_max", "price_avg", "price_unit", "category"]
    # Locate and load the image
    img_path = "img/{0}".format(auction_filename)
    img = Image.open(img_path)
    # Apply OCR routine to image
    text = pytesseract.image_to_string(img)
    text_lines = text.split("\n")
    # Loop through each line and prepare dataframe to append
    for line in text_lines:
        # Check if line corresponds to an iron category or a data entry
        output_is_iron_category = is_iron_category(line)
        output_is_data_entry = is_data_entry(line)
        # Set the iron_category variable if the line describes an iron category
        if output_is_iron_category[0] == True: iron_category = output_is_iron_category[1]
        # Append to data to data array if line is a data entry
        if output_is_data_entry[0] == True: 
            # Construct formatted line
            formatted_line = formatted_data_line(output_is_data_entry[1], output_is_data_entry[2], date, iron_category)
            # Append line if it is not None
            if formatted_line != None: data.append(formatted_line)
    # Create temp dataframe and append it to main dataframe
    temp_df = pd.DataFrame(data=data, columns=columns)
    df = df.append(temp_df)
    # Return the last iron category found in the text
    return iron_category
# Dataframe
df = pd.DataFrame([])
# Initialize progress bar
bar = progressbar.ProgressBar()

# Count to limit number of files for testing
count = -1

# Loop through each auction and process the image data into the dataframe
for auction_ary in bar(auction_filenames):
    # Get date str: YY-MM-DD
    date_str = "-".join(auction_ary[0].replace(".jpg", "").split("-")[0:3]) 
    # Construct date object
    year, month, day = map(int, date_str.split("-"))
    year = year + 2000
    date = datetime.date(year, month, day)
    ####################################
    '''final_count = 1
    # Break on the count file iteration
    count += 1
    if count == final_count:
        break'''
    ####################################
    # Category of livestock: Number of iron brands: 1st male, 2nd male, 1st female, 2nd female.
    iron_category = None
    # Auction filename
    for auction_filename in auction_ary:
        iron_category = append_image_to_dataframe(auction_filename, iron_category, date)

print("#### Error Analysis ####")
print("-------------------------------------")
print("Faulty lines: {0}".format(faulty_lines))
print("Total lines: {0}".format(total_lines))
print("=====================================")
print("Percentage of faults: {0:.2f}%".format(faulty_lines*100/total_lines))<jupyter_output>0% (1 of 458) |                         | Elapsed Time: 0:00:04 ETA:  0:34:46
{'fraction_non_alphanumeric': 0.11519220642443391, 'fraction_numerical': 0.010531858873091101, 'mean_word_length': 3.356077981651376, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '643461', 'license': 'MIT', 'n_tokens_mistral': 2341, 'n_tokens_neox': 2251, 'n_words': 822, 'path': 'dataframe_creation.ipynb', 'repo_name': 'aukintux/livestock_market_analysis', 'size': '7596'}