<jupyter_start><jupyter_text>PIC 2 RECIPE M.Tech Final year project  1. Downloading and Extracting Data<jupyter_code># Downloading the data from MIT source



# Extracting the data<jupyter_output><empty_output><jupyter_text>Observation :    -Size of the Train Data is nearly 95Gb.    -Size of the Validation Data is nearly 15Gb.    -Size of the Test Data is nearly 20Gb.        -In each folder it has Sub folders.    -Each image is having coded name.    -Each image is in different Dimensions.  1.1 Downloading Layers, ingredients and nutritions.<jupyter_code>#!wget -N http://wednesday.csail.mit.edu/temporal/release/det_ingrs.json
#!wget -N http://wednesday.csail.mit.edu/temporal/release/recipe1M_layers.tar.gz
#!wget -N http://wednesday.csail.mit.edu/temporal/release/recipes_with_nutritional_info.json
!wget -N http://wednesday.csail.mit.edu/temporal/release/recipe1M_layers.tar.gz
!tar -xvf recipe1M_layers.tar.gz<jupyter_output><empty_output><jupyter_text>2. Uploading Dataset<jupyter_code>%%time

import os
import re
import copy
import time
import io
import struct
from time import gmtime, strftime

import boto3
import sagemaker
role = sagemaker.get_execution_role()

region = boto3.Session().region_name

bucket = sagemaker.Session().default_bucket()   # Default S3 URI. Replace it with your S3 URI if you want.
prefix = 'sagemaker/recipe1M'
boto3.Session().resource('s3').Bucket(bucket).Object(prefix+'/layer1.json').upload_file('path_to_dataset/layer1.json')
boto3.Session().resource('s3').Bucket(bucket).Object(prefix+'/layer2.json').upload_file('path_to_dataset/layer2.json')
boto3.Session().resource('s3').Bucket(bucket).Object(prefix+'/det_ingrs.json').upload_file('path_to_dataset/det_ingrs.json')
import os
from os import listdir
from os.path import isfile, join
from os import walk

def upload_images(path):
    for i in listdir(path):
        append1 = path + '/' + i + '/'
        for j in listdir(append1):
            append2 = path + '/' + i + '/' + j + '/'
            for k in listdir(append2):
                append3 = path + '/' + i + '/' + j + '/' + k
                for l in listdir(append3):
                    append4 = path + '/' + i + '/' + j + '/' + k + '/' + l
                    for m in listdir(append4):
                        append5 = path + '/' + i + '/' + j + '/' + k + '/' + l + '/' + m
                        for n in listdir(append5):
                            append6 = path + '/' + i + '/' + j + '/' + k + '/' + l + '/' + m + '/' + n
                            locatn = '/'+ i + '/' + j + '/' + k + '/' + l + '/' + m + '/' + n
                            boto3.Session().resource('s3').Bucket(bucket).Object(prefix+locatn).upload_file(append6)

upload_images('path_to_dataset/images/')<jupyter_output><empty_output><jupyter_text>downloading from s3<jupyter_code>%%time

import os
import re
import copy
import time
import io
import struct
from time import gmtime, strftime

import boto3
import sagemaker
role = sagemaker.get_execution_role()

region = boto3.Session().region_name

bucket = sagemaker.Session().default_bucket()   # Default S3 URI. Replace it with your S3 URI if you want.
prefix = 'sagemaker/recipe1M'

bucketname = bucket # replace with your bucket name
filename = 'sagemaker/recipe1M/image_data.csv' # replace with your object key
s3 = boto3.resource('s3')
s3.Bucket(bucketname).download_file(filename, 'image_data.csv')
import numpy as np
import pandas as pd
data = pd.read_csv('conciced_data.csv')<jupyter_output><empty_output><jupyter_text>Collecting the data<jupyter_code>from shutil import copyfile
import os
from os import listdir
from os.path import isfile, join
from os import walk
import pandas as pd

df = pd.read_csv('image_data.csv')
df.drop(['Unnamed: 0'],axis=1, inplace=True)
df
df['partition'].unique()
from shutil import copyfile
import os
from os import listdir
from os.path import isfile, join
from os import walk
def move_images():
    count = 0
    for row in range(0,402760):
        partition = df['partition'][row]
        id1 = df['id'][row]
        id2 = df['image_id'][row]
        
        src = 'path_to_dataset/images/' + partition + '/'  +id2[0] + '/' + id2[1] + '/'  + id2[2] + '/'  + id2[3] + '/'  + id2
        dst = partition + '/' + id1 + "_" +id2
        
        copyfile(src, dst)
        count = count + 1
        if(count % 1000 == 0):
            print('moved ----'+ str(count))                        
move_images()
from PIL import Image 
import os, sys
def resize(path):
    count = 0
    dirs = os.listdir( path )
    for item in dirs:
        if os.path.isfile(path + '/' + item):
            im = Image.open(path + '/' + item)
            imResize = im.resize((256,256), Image.ANTIALIAS)
            imResize.save(path + '1/' + item, 'JPEG', quality=90)
            count = count+1
            if(count%1000 == 0):
                print('resized --- ' + str(count))
    print('total -- ' + str(count) + 'completed!')

resize('train')
resize('val')
resize('test')
from zipfile import ZipFile
zipObj = ZipFile('train.zip', 'w')
count = 0
for each_img in listdir('train1'):
    zipObj.write('train1/'+each_img)
    count = count + 1
    if(count % 1000 == 0):
        print("zipped files ---"+str(count))
print("total files = " + str(count))

zipObj = ZipFile('val.zip', 'w')
count = 0
for each_img in listdir('val1'):
    zipObj.write('val1/'+each_img)
    count = count + 1
    if(count % 1000 == 0):
        print("zipped files ---"+str(count))
print("total files = " + str(count))

zipObj = ZipFile('test.zip', 'w')
count = 0
for each_img in listdir('test1'):
    zipObj.write('test1/'+each_img)
    count = count + 1
    if(count % 1000 == 0):
        print("zipped files ---"+str(count))
print("total files = " + str(count))<jupyter_output>zipped files ---1000
zipped files ---2000
zipped files ---3000
zipped files ---4000
zipped files ---5000
zipped files ---6000
zipped files ---7000
zipped files ---8000
zipped files ---9000
zipped files ---10000
zipped files ---11000
zipped files ---12000
zipped files ---13000
zipped files ---14000
zipped files ---15000
zipped files ---16000
zipped files ---17000
zipped files ---18000
zipped files ---19000
zipped files ---20000
zipped files ---21000
zipped files ---22000
zipped files ---23000
zipped files ---24000
zipped files ---25000
zipped files ---26000
zipped files ---27000
zipped files ---28000
zipped files ---29000
zipped files ---30000
zipped files ---31000
zipped files ---32000
zipped files ---33000
zipped files ---34000
zipped files ---35000
zipped files ---36000
zipped files ---37000
zipped files ---38000
zipped files ---39000
zipped files ---40000
zipped files ---41000
zipped files ---42000
zipped files ---43000
zipped files ---44000
zipped files ---45000
zipped files ---460[...]
{'fraction_non_alphanumeric': 0.14610725365637464, 'fraction_numerical': 0.050967646624316736, 'mean_word_length': 2.893041978148361, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '405943', 'license': 'MIT', 'n_tokens_mistral': 2485, 'n_tokens_neox': 2255, 'n_words': 622, 'path': 'aws/data_handling - Copy.ipynb', 'repo_name': 'Balaganesh-Sarvasuddi/PICTURE-TO-RECIPE-AND-ITS-NUTRITIONAL-INFORMATION-USING-CNN-AND-K-NN', 'size': '6769'}