<jupyter_start><jupyter_text>Data ProcessingThe project has five steps:- delet irregular (too large or small (no data)) and non-image data- remove duplicate image- remove irrelevant image- split dataset: create classes.txt, train.txt, test.txt- rename images  Deleting irragular images<jupyter_code>import os
import sys
import imghdr

class ImageDelet():
    def __init__(self):
        self.path = '/home/gpu/Project/dataProcess/bun/'
        self.imageTypes = ['.jpg', '.jpeg', '.png', '.gif']
        
    delet_count = 0
    
    def delet(self):
        filelist = os.listdir(self.path)
        total_num = len(filelist)

        delet_count = 0

        for item in filelist:
            src = os.path.join(os.path.abspath(self.path), item)
            image_type = os.path.splitext(src)[-1]

            if not imghdr.what(src):  
                os.remove(src)  # delet corrupted image
                delet_count += 1 
            elif image_type in self.imageTypes:
                imageSize = sys.getsizeof(src) # most abnormal image's getsizeof will exceed 150
#                 print(imageSize)
                if  imageSize > 150:
                    os.remove(src)
                    delet_count += 1 
                else:
                    continue
            else:
                os.remove(src)  # delet non-image data 
                delet_count += 1 
        print ('Total: %d\nDelet: %d' % (total_num, delet_count))
deletImage = ImageDelet()
deletImage.delet()<jupyter_output>Total: 19
Delet: 0<jupyter_text>Renaming  the images downloaded by the web crawler.  Renaming the images which have been processed.<jupyter_code>class ImageRename():
    def __init__(self):
        self.path = '/home/gpu/Project/dataProcess/bun/'

    def rename(self):
        filelist = os.listdir(self.path)
        filelist.sort() # if the filelist is not sorted, some file will be replaced when repeating rename result in 
        total_num = len(filelist)

        rename_count = 0

        for item in filelist:
            src = os.path.join(os.path.abspath(self.path), item)
            image_type = os.path.splitext(src)[-1]

#             if image_type in self.imageTypes:
#                 dst = os.path.join(os.path.abspath(self.path), format(str(rename_count), '0>4s') + '.jpg')
            dst = os.path.join(os.path.abspath(self.path), str(rename_count).zfill(4) + image_type)
            os.rename(src, dst)
            print ('converting %s to %s ...' % (src, dst))
            rename_count += 1
#             elif os.path.isdir(src):
#                 continue
#             else:
#                 os.remove(src)
#                 delet_count += 1
                
        print ('Total: %d\nRename: %d' % (total_num, rename_count))
newName = ImageRename()
newName.rename()<jupyter_output>converting /home/gpu/Project/dataProcess/bun/0000.jpeg to /home/gpu/Project/dataProcess/bun/0000.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0001.jpeg to /home/gpu/Project/dataProcess/bun/0001.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0002.jpeg to /home/gpu/Project/dataProcess/bun/0002.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0003.jpeg to /home/gpu/Project/dataProcess/bun/0003.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0004.jpeg to /home/gpu/Project/dataProcess/bun/0004.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0005.jpeg to /home/gpu/Project/dataProcess/bun/0005.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0006.jpeg to /home/gpu/Project/dataProcess/bun/0006.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0007.jpeg to /home/gpu/Project/dataProcess/bun/0007.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0008.jpeg to /home/gpu/Project/dataProcess/bun/0008.jpeg ...
converting /home/gpu/Project/dataProcess/bun/0[...]<jupyter_text>Removing the duplicate images<jupyter_code># Perceptual Hash Algorithm -  dHash
import cv2

def dhash(image):
    # convert image to 8*8
    image = cv2.resize(image, (9, 8), interpolation=cv2.INTER_CUBIC)
    # convert image to grayscale
    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    dhash_str = ''
    for i in range(8):
        for j in range(8):
            if gray[i, j] > gray[i, j + 1]:
                dhash_str = dhash_str + '1'
            else:
                dhash_str = dhash_str + '0'
    result = ''
    for i in range(0, 64, 4):
        result += ''.join('%x' % int(dhash_str[i: i + 4], 2))
    return result

# calculate the difference between hash1 and hash2
def campHash(hash1, hash2):
    n = 0
    # If the hash length is different, the comparison cannot be made, and -1 is returned.
    if len(hash1) != len(hash2):
        return -1
    # If the hash length is same, traversing hash1 ahd hash2 for comparison.
    for i in range(len(hash1)):
        if hash1[i] != hash2[i]:
            n = n + 1
    return n
image1 = cv2.imread('/home/gpu/Project/dataProcess/bun/0017.jpg')
image2 = cv2.imread('/home/gpu/Project/dataProcess/bun/0018.jpeg')

hash1 = dhash(image1)
hash2 = dhash(image2)

distance_hash = campHash(hash1, hash2)

# if campHash == 0, it means that the two images are duplicate images.
image2_path = '/home/gpu/Project/dataProcess/bun/0012.jpeg'
if distance_hash == 0:
    os.remove(image2_path)<jupyter_output><empty_output><jupyter_text>Removing the irrelevant  images<jupyter_code># Perceptual Hash Algorithm -  dHash
import cv2

def dhash(image):
    # convert image to 8*8
    image = cv2.resize(image, (9, 8), interpolation=cv2.INTER_CUBIC)
    # convert image to grayscale
    gray = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
    dhash_str = ''
    for i in range(8):
        for j in range(8):
            if gray[i, j] > gray[i, j + 1]:
                dhash_str = dhash_str + '1'
            else:
                dhash_str = dhash_str + '0'
    result = ''
    for i in range(0, 64, 4):
        result += ''.join('%x' % int(dhash_str[i: i + 4], 2))
    return result

# calculate the difference between hash1 and hash2
def campHash(hash1, hash2):
    n = 0
    #  If the hash length is different, the comparison cannot be made, and -1 is returned.
    if len(hash1) != len(hash2):
        return -1
    # If the hash length is same, traversing hash1 ahd hash2 for comparison.
    for i in range(len(hash1)):
        if hash1[i] != hash2[i]:
            n = n + 1
    return n
image1 = cv2.imread('/home/gpu/Project/dataProcess/bun/0017.jpg')
image2 = cv2.imread('/home/gpu/Project/dataProcess/bun/0013.jpeg')

hash1 = dhash(image1)
hash2 = dhash(image2)

distance_hash = campHash(hash1, hash2)

# if campHash > 10, it means that the two images are different classes.
image2_path = '/home/gpu/Project/dataProcess/bun/0012.jpeg'
if distance_hash > 10:
    os.remove(image2_path)<jupyter_output><empty_output><jupyter_text>Spliting dataset  Generate the train.txt, test.txt, and  classes.txt.<jupyter_code>dataset_path = '/home/gpu/Project/dataProcess/'

def gengrateClass(dataset_path):
    filelist = os.listdir(dataset_path)
    
    for file_name in filelist:
        if file_name.startswith('.'):
            filelist.remove(file_name)
    
    filelist.sort()
    
    class_savePath = '/home/gpu/Project/dataProcess/meta/class.txt'
    
    # If filename does not exist, it will be created automatically. 
    #'w' means to write data. The original data in the file will be cleared before writing!
    with open(class_savePath,'w') as f:
        for file_name in filelist:
            f.write(file_name)
            f.write('\n')
   

gengrateClass(dataset_path)
import math
def splitDataset(dataset_path):
    filelist = os.listdir(dataset_path)
    
    for file_name in filelist:
        if file_name.startswith('.'):
            filelist.remove(file_name)
            
    filelist.sort()
    
    train_savePath = '/home/gpu/Project/dataProcess/meta/train.txt'
    test_savePath = '/home/gpu/Project/dataProcess/meta/test.txt'
    
    for file_name in filelist:
        image_path = dataset_path + file_name
        image_list = os.listdir(image_path)
        
        for image_name in image_list:
            if image_name.startswith('.'):
                image_list.remove(image_name)

        image_size = len(image_list)
        train_size = math.ceil(image_size * 0.75)
        
        # If filename does not exist, it will be created automatically. 
        #'w' means to append data. The original data in the file will not be cleared.
        with open(train_savePath,'a') as train:
            for file_name in image_list[:train_size]:
                train.write(file_name)
                train.write('\n')
        with open(test_savePath,'a') as test:
            for file_name in image_list[train_size:]:
                test.write(file_name)
                test.write('\n')

splitDataset(dataset_path)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11589778380823157, 'fraction_numerical': 0.02532790592492085, 'mean_word_length': 2.4864012613322823, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '11431', 'license': 'MIT', 'n_tokens_mistral': 3074, 'n_tokens_neox': 2879, 'n_words': 693, 'path': '2.dataProcessing/dataProcessing.ipynb', 'repo_name': 'QiujieDong/ChineseFood', 'size': '8844'}