<jupyter_start><jupyter_text>Data Preparation<jupyter_code>#importing the dependencies
import os
import cv2
import numpy as np
import matplotlib.pyplot as plt
CAPTCHA_IMAGE_FOLDER = "generated_captcha_images"
OUTPUT_FOLDER = "extracted_letter_images"
#forming the train and test directories
image_names=os.listdir(CAPTCHA_IMAGE_FOLDER)
test_image_names=image_names[:100]
train_image_names=image_names[100:]
os.mkdir(CAPTCHA_IMAGE_FOLDER+'/'+'train_images')
os.mkdir(CAPTCHA_IMAGE_FOLDER+'/'+'test_images')
for image in train_image_names:
    os.rename(CAPTCHA_IMAGE_FOLDER+'/'+image,CAPTCHA_IMAGE_FOLDER+'/'+'train_images/'+image)
for image in test_image_names:
    os.rename(CAPTCHA_IMAGE_FOLDER+'/'+image,CAPTCHA_IMAGE_FOLDER+'/'+'test_images/'+image)<jupyter_output><empty_output><jupyter_text>Forming our data set of single characters using contours to segment the CAPTCHA letters<jupyter_code>def preprocess(image_names):
    letter_count={}
    i=1
    CAPTCHA_IMG_FOLDER=CAPTCHA_IMAGE_FOLDER+'/'+'train_images'
    for image in image_names:
        print("images processed: ",i,"out of ",len(image_names))
        i+=1
        if(image.startswith('.')):
            continue
        im_path=CAPTCHA_IMG_FOLDER +'/'+ image
        img=cv2.imread(im_path) #reading the image
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) #converting to gray scale
        gray = cv2.copyMakeBorder(gray, 8, 8, 8, 8, cv2.BORDER_REPLICATE) # Add some extra padding around the image
        thresh = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)[1]  # threshold the image (convert it to pure black and white)
        contours= cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # find the contours (continuous blobs of pixels) the image
        contours=contours[0]
        im_letters=[]
        for contour in contours: # Now we can loop through each of the four contours and extract the letter inside of each one
            (x,y,w,h)=cv2.boundingRect(contour)
            if w/h>1.5: #if contour to wide the split it into 2 
                im_letters.append((x,y,w//2,h))
                im_letters.append((x+w//2,y,w//2,h))
            else:
                im_letters.append((x,y,w,h))
        if(len(im_letters)!=4):
            continue
        im_letters=sorted(im_letters,key=lambda x : x[0]) #Sort the detected letter images based on the x coordinate to make sure we are processing them from left-to-right
        capcha=str(image).split('.')[0]
        for letter_box,letter in zip(im_letters,capcha): # Save out each letter as a single image
            (x,y,w,h)=letter_box
            letter_img=gray[y-2:y+h+2,x-2:x+w+2] # Extract the letter from the original image with a 2-pixel margin around the edge
            l_count=letter_count.get(letter,1)
            l_imfolder_path=OUTPUT_FOLDER + '/' +letter
            l_img_path=l_imfolder_path+'/'+str(l_count).zfill(6)+'.png'
            if(letter not in os.listdir(OUTPUT_FOLDER)):
                os.mkdir(l_imfolder_path)
            cv2.imwrite(l_img_path, letter_img) # write the letter image to a file
            letter_count[letter]=l_count+1
preprocess(train_image_names)<jupyter_output>images processed:  1 out of  9855
images processed:  2 out of  9855
images processed:  3 out of  9855
images processed:  4 out of  9855
images processed:  5 out of  9855
images processed:  6 out of  9855
images processed:  7 out of  9855
images processed:  8 out of  9855
images processed:  9 out of  9855
images processed:  10 out of  9855
images processed:  11 out of  9855
images processed:  12 out of  9855
images processed:  13 out of  9855
images processed:  14 out of  9855
images processed:  15 out of  9855
images processed:  16 out of  9855
images processed:  17 out of  9855
images processed:  18 out of  9855
images processed:  19 out of  9855
images processed:  20 out of  9855
images processed:  21 out of  9855
images processed:  22 out of  9855
images processed:  23 out of  9855
images processed:  24 out of  9855
images processed:  25 out of  9855
images processed:  26 out of  9855
images processed:  27 out of  9855
images processed:  28 out of  9855
images processed:  29 out of [...]
{'fraction_non_alphanumeric': 0.09, 'fraction_numerical': 0.05023809523809524, 'mean_word_length': 3.586244541484716, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '201336', 'license': 'MIT', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1419, 'n_words': 408, 'path': '.ipynb_checkpoints/capcha_data_preprocessing-checkpoint.ipynb', 'repo_name': 'AmritK10/CAPTCHA_Text_Recognition', 'size': '4200'}