<jupyter_start><jupyter_text>손글씨 데이터에 대해 rawdata 및 전처리, augmentation 수행한 최종 데이터 얻음  손글씨 데이터 파악<jupyter_code>import json
def getListOfHandWritingFileNames(listOfFemaleHandWritingFileNames, listOfMaleHandWritingFileNames):
    handwritingMetaFile_location = '/content/gdrive/Shareddrives/machine_learning_in_practice/한국어 글자체 이미지/01.손글씨/handwriting_data_info1.json'

    with open(handwritingMetaFile_location, 'r') as f:
        hw_json_data = json.load(f)

    hw_ClassToFileNames, hw_WordsToFileNames= {}, {}
    hw_FileNameToClass = {}
    hw_syllableDataLength = 0

    for idx, img_data in enumerate(hw_json_data['images']):
        # idx 번째 이미지 데이터들에 대해 {클래스 : [파일명, 파일명, ...]}
        class_ = hw_json_data['annotations'][idx]['text']
        fileName = hw_json_data['images'][idx]['file_name']
        if len(class_) == 1:    
            # 한 글자인 경우만 ClassToFileNames에 넣기
            if class_ not in hw_ClassToFileNames.keys(): # idx번째 이미지의 클래스가 새로운 클래스이면
                hw_ClassToFileNames[class_] = [] # 빈 리스트 생성
            hw_ClassToFileNames[class_].append(fileName)
            hw_syllableDataLength += 1
            if hw_json_data['annotations'][1]['attributes']['gender'] == '여':
                listOfFemaleHandWritingFileNames.append(fileName)
            else:
                listOfMaleHandWritingFileNames.append(fileName)
        else:
            # 단어, 문장의 경우 WordsToFileNames에 넣기
            if class_ not in hw_WordsToFileNames.keys(): # idx번째 이미지의 클래스가 새로운 클래스이면
                hw_WordsToFileNames[class_] = [] # 빈 리스트 생성
            hw_WordsToFileNames[class_].append(fileName)
    
    for idx, img_data in enumerate(hw_json_data['images']):
        fileName = hw_json_data['images'][idx]['file_name']
        class_ = hw_json_data['annotations'][idx]['text']
        hw_FileNameToClass[fileName] = class_
    
    print("*** Handwriting Data Description ***")
    print("\t전체 데이터 수 : ", len(hw_json_data['images']) )# 여기에는 문장 데이터, 단어 데이터, 음절 데이터 모두 포함됨
    print("\t음절 클래스 수 : ", len(hw_ClassToFileNames) ) # 한국어 글자 종류는 11,172가지.
    print("\t단어/문장 클래스 수: ", len(hw_WordsToFileNames))
    print("\t음절 데이터 수 : ", hw_syllableDataLength)
    print("\t단어/문장 데이터 수 : ", len(hw_json_data['images']) - hw_syllableDataLength)
    print(f"\t한 음절 당 평균 {hw_syllableDataLength/len(hw_ClassToFileNames)} 개 손글씨 데이터 존재함")

    return hw_ClassToFileNames, hw_FileNameToClass
listOfFemaleHandWritingFileNames, listOfMaleHandWritingFileNames = [], []

hw_ClassToFileNames, hw_FileNameToClass = getListOfHandWritingFileNames(listOfFemaleHandWritingFileNames, listOfMaleHandWritingFileNames)

print("여성 필기체 data 수 : ", len(listOfFemaleHandWritingFileNames))
print("남성 필기체 data 수 : ", len(listOfMaleHandWritingFileNames))<jupyter_output>*** Handwriting Data Description ***
	전체 데이터 수 :  1170622
	음절 클래스 수 :  11172
	단어/문장 클래스 수:  6054
	음절 데이터 수 :  761109
	단어/문장 데이터 수 :  409513
	한 음절 당 평균 68.1264769065521 개 손글씨 데이터 존재함
여성 필기체 data 수 :  761109
남성 필기체 data 수 :  0<jupyter_text>newClassToFileName<jupyter_code>import csv 
path = '/content/gdrive/Shareddrives/machine_learning_in_practice/Analog-PILGI-to-DIgital/GAN/data/pretrain/classes.csv'
f = open(path,'r')
rdr = csv.reader(f)
for line in rdr:
    classes = line 
f.close()

newClassToFileNames = {} # key 1268개
for syllable in classes:
    newClassToFileNames[syllable] = hw_ClassToFileNames[syllable]<jupyter_output><empty_output><jupyter_text>FileNameToClass.pickle - key를 한정 (기존 FileNameToClass는 11172개의 class 모두 담고 있음 => 1268개 클래스로 축소)<jupyter_code>classes = newClassToFileNames.keys()
new_hwFileNameToClass = dict()
for fileName in hw_FileNameToClass.keys():
    if hw_FileNameToClass[fileName] not in classes:
        continue
    new_hwFileNameToClass[fileName] = hw_FileNameToClass[fileName]

print("클래스 축소 전 데이터 수 : ", len(hw_FileNameToClass))
print("클래스 축소 후 데이터 수 : ", len(new_hwFileNameToClass))<jupyter_output>클래스 축소 전 데이터 수 :  1170622
클래스 축소 후 데이터 수 :  106076<jupyter_text>raw_handwritten.zip | 1268개 클래스에 해당하는 이미지 저장<jupyter_code>!unzip '/content/gdrive/Shareddrives/machine_learning_in_practice/한국어 글자체 이미지/01.손글씨/01_handwriting_syllable_images.zip' -d .
import os

path = '/content/1_syllable'
file_list = os.listdir(path)

# new_pFileNameToClass 는 1268개 클래스에 해당하는 image filename 들만 key 로 있음
newFileList = []
newClassList = []
for filename in file_list:
    try: # new_pFileNameToClass 이용함 key 에러 뜨면 넘겨야 함
        class_ = new_hwFileNameToClass[filename]
        newFileList.append(filename)
        newClassList.append(class_)
    except KeyError:
        # print(filename)
        pass
print(len(newFileList))
print(len(newClassList))
import csv

filepath = '/content/gdrive/Shareddrives/machine_learning_in_practice/Analog-PILGI-to-DIgital/GAN/data/transfer/fileName.csv'
with open(filepath, 'w', newline='') as f: 
    writer = csv.writer(f) 
    writer.writerow(newFileList)
import os
import zipfile

new_zips= zipfile.ZipFile('raw_handwritten.zip', 'w') # 
num_zipfile = 0

for folder, subfolders, files in os.walk('/content/1_syllable'): 
    for file in files:
        try:
            new_hwFileNameToClass[file] # 1268 클래스에 속하는 데이터만 zip함  총 112300개
            new_zips.write(os.path.join(folder, file), os.path.relpath(os.path.join(folder,file), './'), compress_type = zipfile.ZIP_DEFLATED)
            num_zipfile += 1
        except KeyError:
            pass
new_zips.close()
print(num_zipfile)
import shutil

savePath = '/content/gdrive/Shareddrives/machine_learning_in_practice/Analog-PILGI-to-DIgital/GAN/data/transfer'

shutil.move('/content/raw_handwritten.zip', savePath)<jupyter_output><empty_output><jupyter_text>croppedImg 생성<jupyter_code>!unzip '/content/gdrive/Shareddrives/machine_learning_in_practice/Analog-PILGI-to-DIgital/GAN/data/transfer/raw_handwritten.zip' -d ./rawdata
import cv2
import csv
import os
import numpy as np
from PIL import Image, ImageOps
import matplotlib.image as img 

savePath = '/content/preprocessed' 

path = '/content/rawdata/1_syllable'
file_list_ = os.listdir(path)

for filename in file_list_:
    filePath = path + '/' + filename
    image=Image.open(filePath)
    image.load()
    imageSize = image.size
    
    # remove alpha channel
    invert_im = image.convert("RGB") 
    
    # invert image (so that white is 0)
    invert_im = ImageOps.invert(invert_im)
    imageBox = invert_im.getbbox()
    
    cropped=image.crop(imageBox)
    print(filePath, "Size:", imageSize, "New Size:", imageBox)
    # cropped.save(savePath + '/' + filename)
    break
import os
import zipfile

new_zips= zipfile.ZipFile('preprocessed.zip', 'w')
 
for folder, subfolders, files in os.walk('/content/preprocessed/'): 
    for file in files:
        new_zips.write(os.path.join(folder, file), os.path.relpath(os.path.join(folder,file), './'), compress_type = zipfile.ZIP_DEFLATED)

new_zips.close()
import shutil

savePath = '/content/gdrive/Shareddrives/machine_learning_in_practice/Analog-PILGI-to-DIgital/GAN/data/transfer'

shutil.move('preprocessed.zip', savePath)<jupyter_output><empty_output><jupyter_text>github 커밋<jupyter_code>MY_GOOGLE_DRIVE_PATH = "/content/gdrive/Shareddrives/machine_learning_in_practice/Analog-PILGI-to-DIgital"
%cd "{MY_GOOGLE_DRIVE_PATH}"
!git config --global user.email <EMAIL>  # 이메일 입력 ex) <EMAIL>
!git config --global user.name  hyeneung #깃헙 아이디 입력 ex)luckydipper
!git pull
!git status
!git add GAN/PrintedDataset.ipynb GAN/Parsing.ipynb GAN/HandWrittenDataset.ipynb
!git commit -m"[ADD] zip handwrittenDataset"
!git push<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10919692552345614, 'fraction_numerical': 0.0155049032600053, 'mean_word_length': 4.194081211286992, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '371543', 'license': 'MIT', 'n_tokens_mistral': 3315, 'n_tokens_neox': 3337, 'n_words': 537, 'path': 'GAN/HandWrittenDataset.ipynb', 'repo_name': 'PILGI-Analog-To-Digital/Analog-PILGI-to-DIgital', 'size': '7571'}