<jupyter_start><jupyter_text>size statistic<jupyter_code>path = '../dataset/pre_data/train/CN'
size_list = []
for item in os.scandir(path):
    img = Image.open(item.path).convert('L')
    size_list.append(img.size)
print(len(size_list))
print('********')
unique_size = set(size_list)
for unique_item in unique_size:
    print(unique_item)
    print(size_list.count(unique_item))<jupyter_output>1000
********
(256, 256)
15
(168, 168)
344
(128, 128)
641<jupyter_text>data visualization<jupyter_code>import glob
import torchvision.transforms as tr
import numpy as np
path = '../dataset/pre_data/train/AD'
path_list = path_list = glob.glob(os.path.join(path,"*.png"))

for item in path_list[:10]:
    img = Image.open(item).convert('L')
    # img = img.resize((128,128),Image.NEAREST)
    print(img.size)
    print(type(img))

    transform = transforms.Compose([
      tr.Resize(size=(128,128)),
      tr.RandomRotation(degrees=30),
      tr.RandomHorizontalFlip(p=0.5),
      tr.RandomVerticalFlip(p=0.5),
      tr.ToTensor()
    ])
    img = transform(img)
    print(type(img))
    print(img.shape)
    print(np.max(img.numpy()))
    plt.imshow(img.numpy()[0],cmap='gray')
    plt.show()
    
#     plt.hist(img)
#     plt.show()<jupyter_output>(128, 128)
<class 'PIL.Image.Image'>
<class 'torch.Tensor'>
torch.Size([1, 128, 128])
1.0
{'fraction_non_alphanumeric': 0.17916041979010494, 'fraction_numerical': 0.049475262368815595, 'mean_word_length': 4.21484375, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '469020', 'license': 'MIT', 'n_tokens_mistral': 556, 'n_tokens_neox': 510, 'n_words': 80, 'path': 'converter/data_visual.ipynb', 'repo_name': 'shijun18/XunFei_Classifier', 'size': '1334'}