<jupyter_start><jupyter_text>fruit dataset<jupyter_code>root = 'data/train'
size = [224,224]
imgs_a = []
imgs_b = []
for subdir in os.listdir(root):
    root_sub = os.path.join(root, subdir)
    for img_name in os.listdir(root_sub):
        root_img = os.path.join(root_sub, img_name)
        img = Image.open(root_img).resize(size)
        img = np.asarray(img)
        img_lab = rgb2lab(img)
        imgs_a.append(img_lab[:,:,1])
        imgs_b.append(img_lab[:,:,2])
a = np.stack(imgs_a, axis=0)
b = np.stack(imgs_b, axis=0)
x = a.reshape([-1,])
y = b.reshape([-1,])
idx_x = (x>110) + (x<-110) + np.isnan(x)
idx_y = (y>110) + (y<-110) + np.isnan(y)
x_new = x[~idx_x]
y_new = y[~idx_x]
print(x.shape)
print(x_new.shape)
print(y_new.shape)
h_range = np.array([[-110,110], [-110,110]])
h, xaxis, yaxis = np.histogram2d(x_new, y_new, bins=22, range=h_range, density=True)
X,Y = np.meshgrid(xaxis, yaxis)
plt.pcolormesh(X, Y, np.log(h.T))
plt.colorbar()
plt.title('logP(a,b), fruit dataset')
plt.xlabel('a')
plt.ylabel('b')
plt.show()<jupyter_output>C:\Users\Hao\AppData\Local\Temp/ipykernel_21160/3872900061.py:4: RuntimeWarning: divide by zero encountered in log
  plt.pcolormesh(X, Y, np.log(h.T))<jupyter_text>\art_images\musemart\training_set\iconography<jupyter_code>root = 'data/art_images/musemart/training_set/iconography/total'
size = [224,224]
imgs_a = []
imgs_b = []
for img_name in os.listdir(root):
    root_img = os.path.join(root, img_name)
    img = Image.open(root_img).resize(size)
    img = np.asarray(img)
    img_lab = rgb2lab(img)
    imgs_a.append(img_lab[:,:,1])
    imgs_b.append(img_lab[:,:,2])
a = np.stack(imgs_a, axis=0)
b = np.stack(imgs_b, axis=0)
x = a.reshape([-1,])
y = b.reshape([-1,])
idx_x = (x>110) + (x<-110) + np.isnan(x)
idx_y = (y>110) + (y<-110) + np.isnan(y)
x_new = x[~idx_x]
y_new = y[~idx_x]
print(x.shape)
print(x_new.shape)
print(y_new.shape)
h_range = np.array([[-110,110], [-110,110]])
h, xaxis, yaxis = np.histogram2d(x_new, y_new, bins=22, range=h_range, density=True)
X,Y = np.meshgrid(xaxis, yaxis)
plt.pcolormesh(X, Y, np.log(h.T))
plt.colorbar()
plt.title('logP(a,b), art image dataset')
plt.xlabel('a')
plt.ylabel('b')
plt.show()<jupyter_output>C:\Users\Hao\AppData\Local\Temp/ipykernel_21160/232295639.py:4: RuntimeWarning: divide by zero encountered in log
  plt.pcolormesh(X, Y, np.log(h.T))<jupyter_text>VOCdataset\test<jupyter_code>root = 'data/VOCdataset/test'
size = [224,224]
a = np.empty(size)
b = np.empty(size)
for img_name in os.listdir(root):
    root_img = os.path.join(root, img_name)
    img = Image.open(root_img).resize(size)
    img = np.asarray(img)
    img_lab = rgb2lab(img)
    a = np.concatenate((a, img_lab[:,:,1]), axis=0)
    b = np.concatenate((b, img_lab[:,:,2]), axis=0)

x = a.reshape([-1,])
y = b.reshape([-1,])
idx_x = (x>110) + (x<-110) + np.isnan(x)
idx_y = (y>110) + (y<-110) + np.isnan(y)
x_new = x[~idx_x]
y_new = y[~idx_x]
print(x.shape)
print(x_new.shape)
print(y_new.shape)

h_range = np.array([[-110,110], [-110,110]])
h, xaxis, yaxis = np.histogram2d(x_new, y_new, bins=22, range=h_range, density=True)
X,Y = np.meshgrid(xaxis, yaxis)

plt.pcolormesh(X, Y, np.log(h.T))
plt.colorbar()
plt.title('logP(a,b)')
plt.show()<jupyter_output>C:\Users\Hao\AppData\Local\Temp/ipykernel_14676/2837664527.py:1: RuntimeWarning: divide by zero encountered in log
  plt.pcolormesh(X, Y, np.log(h.T))<jupyter_text>make grid<jupyter_code>import torchvision
import torch
from torchvision import transforms
from matplotlib import pyplot as plt<jupyter_output><empty_output><jupyter_text>fruit<jupyter_code>data_transforms = transforms.Compose([transforms.Resize([256,256]),transforms.ToTensor()])
fruit_dataset = torchvision.datasets.ImageFolder(root='data/train', transform=data_transforms)
fruit_loader = torch.utils.data.DataLoader(fruit_dataset, batch_size=32, shuffle=True)
imgs = next(iter(fruit_loader))
print(imgs[0].shape)
img_grid = torchvision.utils.make_grid(imgs[0])
# print(img_grid.shape)
img_grid = img_grid.permute([1,2,0])
plt.figure(figsize=[8,20])
plt.imshow(img_grid)
plt.axis('off')<jupyter_output>torch.Size([3, 1034, 2066])<jupyter_text>iconography<jupyter_code>data_transforms = transforms.Compose([transforms.Resize([256,256]),transforms.ToTensor()])
fruit_dataset = torchvision.datasets.ImageFolder(root='data/art_images/musemart/training_set/iconography', 
                                                 transform=data_transforms)
fruit_loader = torch.utils.data.DataLoader(fruit_dataset, batch_size=32, shuffle=True)
imgs = next(iter(fruit_loader))
img_grid = torchvision.utils.make_grid(imgs[0],padding=2)
img_grid = img_grid.permute([1,2,0])
plt.figure(figsize=[8,20])
plt.imshow(img_grid)
plt.axis('off')<jupyter_output><empty_output><jupyter_text>VOC<jupyter_code>data_transforms = transforms.Compose([transforms.Resize([256,256]),transforms.ToTensor()])
fruit_dataset = torchvision.datasets.ImageFolder(root='data/VOCdataset/test/', 
                                                 transform=data_transforms)
fruit_loader = torch.utils.data.DataLoader(fruit_dataset, batch_size=32, shuffle=True)
imgs = next(iter(fruit_loader))
img_grid = torchvision.utils.make_grid(imgs[0],padding=2)
img_grid = img_grid.permute([1,2,0])
plt.figure(figsize=[8,20])
plt.imshow(img_grid)
plt.axis('off')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17810273715785527, 'fraction_numerical': 0.04199475065616798, 'mean_word_length': 5.490267639902676, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '365669', 'license': 'MIT', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2254, 'n_words': 314, 'path': 'utils.ipynb', 'repo_name': 'zhthuuu/colorization-cnn', 'size': '5334'}