<jupyter_start><jupyter_text>Prepare data<jupyter_code># mount google drive & set working directory
# requires auth (click on url & copy token into text box when prompted)
from google.colab import drive
drive.mount("/content/gdrive", force_remount=True)

import os
print(os.getcwd())

os.chdir('/content/gdrive/My Drive/Colab Notebooks/MidcurveNN')
!pwd
!pip install drawSVG
"""
    Prepare Data: populating input images from raw profile data
    Takes raw data from "data/raw/*" files for both, profile shape (shape.dat) as well as midcurve shape (shape.mid)
    Generates raster image files from svg (simple vector graphics)
    Multiple variations are populated using image transformations.
    These images become input for further modeling (stored in "data/input/*")
"""
import os
import sys
import PIL
import json
import shutil
import numpy as np
import PIL.ImageOps
from random import shuffle
from keras.preprocessing.image import img_to_array, load_img, array_to_img
np.set_printoptions(threshold=sys.maxsize)

from PIL import Image

# working directory
#wdir = os.getcwd()
wdir = '/content/gdrive/My Drive/Colab Notebooks/MidcurveNN'
print("Working directory: ", wdir)
imdim = 100
#input_data_folder = wdir + "\\data\\sample"
#input_data_folder = wdir + "/data/newinput"
#print("input data dir: ", input_data_folder)
raw_data_folder = "data/new_shapes"
input_data_folder = "data/new_images"
pix2pix_data_folder = "/data/pix2pix/datasets/pix2pix"
def read_dat_files(datafolder=raw_data_folder):
    profiles_dict_list = []
    for file in os.listdir(datafolder):
        if os.path.isdir(os.path.join(datafolder, file)):
            continue
        filename = file.split(".")[0]
        profile_dict = get_profile_dict(filename,profiles_dict_list)        
        if file.endswith(".dat"):
            with open(os.path.join(datafolder, file)) as f:
                profile_dict['Profile'] = [tuple(map(float, i.split('\t'))) for i in f]  
        if file.endswith(".mid"):
            with open(os.path.join(datafolder, file)) as f:
                profile_dict['Midcurve'] = [tuple(map(float, i.split('\t'))) for i in f]
                                
        profiles_dict_list.append(profile_dict)
    return profiles_dict_list
  

def get_profile_dict(shapename,profiles_dict_list):
    for i in profiles_dict_list:
        if i['ShapeName'] == shapename:
            return i
    profile_dict = {}
    profile_dict['ShapeName'] = shapename
    return profile_dict


import drawSvg as draw

def create_image_file(fieldname,profile_dict,datafolder=input_data_folder,imgsize=imdim, isOpenClose=True):
    d = draw.Drawing(imgsize, imgsize, origin='center')
    profilepoints = []
    for tpl in profile_dict[fieldname]:
        profilepoints.append(tpl[0])
        profilepoints.append(tpl[1])
    d.append(draw.Lines(profilepoints[0],profilepoints[1],*profilepoints,close=isOpenClose,fill='none',stroke='black'))
    
    shape = profile_dict['ShapeName']
#     d.saveSvg(datafolder+"/"+shape+'.svg')
#    d.savePng(datafolder+"/"+shape+'_'+fieldname+'.png')
    d.savePng(datafolder+"/"+shape+'_'+fieldname+'.png')


def get_original_png_files(datafolder=input_data_folder):
    pngfilenames = []
    for file in os.listdir(datafolder):
        fullpath = os.path.join(datafolder, file)
        if os.path.isdir(fullpath):
            continue
        if file.endswith(".png") and file.find("_rotated_") == -1 and file.find("_translated_")==-1 and file.find("_mirrored_")==-1:
            pngfilenames.append(fullpath)
    return pngfilenames


def mirror_images(pngfilenames, mode=PIL.Image.TRANSPOSE):
    mirrored_filenames = []
    for fullpath in pngfilenames:
        picture= Image.open(fullpath)
        newfilename = fullpath.replace(".png", "_mirrored_"+str(mode)+".png")
        picture.transpose(mode).save(newfilename)
        mirrored_filenames.append(newfilename)
    return mirrored_filenames


def rotate_images(pngfilenames, angle=90):
    for fullpath in pngfilenames:
        picture= Image.open(fullpath)
        newfilename = fullpath.replace(".png", "_rotated_"+str(angle)+".png")
        picture.rotate(angle).save(newfilename)


def translate_images(pngfilenames, dx=1,dy=1):
    for fullpath in pngfilenames:
        picture= Image.open(fullpath)
        x_shift = dx
        y_shift = dy
        a = 1
        b = 0
        c = x_shift #left/right (i.e. 5/-5)
        d = 0
        e = 1
        f = y_shift #up/down (i.e. 5/-5)
        translate = picture.transform(picture.size, Image.AFFINE, (a, b, c, d, e, f))
#         # Calculate the size after cropping
#         size = (translate.size[0] - x_shift, translate.size[1] - y_shift)
#         # Crop to the desired size
#         translate = translate.transform(size, Image.EXTENT, (0, 0, size[0], size[1]))
        newfilename = fullpath.replace(".png", "_translated_"+str(dx)+"_"+str(dy)+".png")
        translate.save(newfilename)

def generate_images(datafolder=input_data_folder):
    
    if not os.path.exists(datafolder):
        os.makedirs(datafolder)    
    else:    
        for file in os.listdir(datafolder):
            if file.endswith(".png") and (file.find("_rotated_") != -1 or file.find("_translated_") !=-1):
                print("files already present, not generating...")
                return
                
    print("transformed files not present, generating...")
    profiles_dict_list = read_dat_files()
    
    print(profiles_dict_list)
    
    for profile_dict in profiles_dict_list:
        create_image_file('Profile',profile_dict,datafolder,imdim,True)
        create_image_file('Midcurve',profile_dict,datafolder,imdim,False)
        
    pngfilenames = get_original_png_files(datafolder)
    mirrored_filenames_left_right = mirror_images(pngfilenames, PIL.Image.FLIP_LEFT_RIGHT)
    mirrored_filenames_top_bottom = mirror_images(pngfilenames, PIL.Image.FLIP_TOP_BOTTOM)
    mirrored_filenames_transpose = mirror_images(pngfilenames, PIL.Image.TRANSPOSE)
    
    files_list_list = [pngfilenames,mirrored_filenames_left_right,mirrored_filenames_top_bottom,mirrored_filenames_transpose]
    for filelist in files_list_list:
        for angle in range(30,360,30):
            rotate_images(filelist,angle)
            
        for dx in range(5,21,5):
            for dy in range(5,21,5):
                translate_images(filelist,dx,-dy)
generate_images()
# wait till all images are generated before executing the next cell
break
# move images to appropriate directories
# directory names follows the shape name

import os
import shutil

srcpath = input_data_folder
destpath = input_data_folder

for root, subFolders, files in os.walk(srcpath):
    for file in files:
        #print(file)
        subFolder = os.path.join(destpath, file[:4])
        if not os.path.isdir(subFolder):
            os.makedirs(subFolder)
        try:
            shutil.move(os.path.join(root, file), subFolder)
        except:
            pass
print(wdir)

# move images from temporary directory to actual
# directory names follows the shape name

src_shapes = wdir + "/data/new_shapes/"
src_images = wdir + "/data/new_images/"

dest_shapes = wdir + "/data/shapes/"
dest_images = wdir + "/data/images/"


files = os.listdir(src_shapes)
for f in files:
    shutil.move(src_shapes+f, dest_shapes)
    
files = os.listdir(src_images)
for f in files:
    shutil.move(src_images+f, dest_images)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11321769530724754, 'fraction_numerical': 0.006723141051499261, 'mean_word_length': 3.3830288744843844, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '5330', 'license': 'MIT', 'n_tokens_mistral': 2467, 'n_tokens_neox': 2396, 'n_words': 523, 'path': 'projects/midcurvenn/prepare_data_1.ipynb', 'repo_name': 'futureseadev/hgwxx7', 'size': '7437'}