<jupyter_start><jupyter_text>calculate mean and variance<jupyter_code>import PIL
from PIL import Image, ImageStat
# change train image directory file path 
train_img_dir = '/content/gdrive/My Drive/train_corrected/train'   # directory of images 
image_stats_list = [ImageStat.Stat(Image.open(train_img_dir + '/' + image_file)) for image_file in image_name.values]
images_red_channel, images_green_channel, images_blue_channel = [stat.mean[0] for stat in image_stats_list], [stat.mean[1] for stat in image_stats_list], [stat.mean[2] for stat in image_stats_list]
red_mean, red_variance = np.mean(images_red_channel), np.var(images_red_channel)
green_mean, green_variance = np.mean(images_green_channel), np.var(images_green_channel)
blue_mean, blue_variance = np.mean(images_blue_channel), np.var(images_blue_channel)
print(red_mean)
print(red_variance)
print(green_mean)
print(green_variance)
print(blue_mean)
print(blue_variance)
# import math library 
import math 
red_std=math.sqrt(red_variance)
green_std=math.sqrt(green_variance)
blue_std=math.sqrt(blue_variance)


#red_mean=126.387
#red_std=36.263
#green_mean=131.316
#red_std=37.283
#blue_mean=144.899
#blue_std=36.225<jupyter_output><empty_output><jupyter_text>convert categorical lables by mapping<jupyter_code>#import test data
# target to numbers 

train_cars=pd.read_csv('/content/gdrive/My Drive/train_corrected/train_cars.csv') # file path of original train_cars.csv
img_name=train_cars['image_name']
target=train_cars['target']

target = target.map({
    "bus": 0,
    "cement mixer": 1,
    "crane truck": 2,
    "dedicated agricultural vehicle": 3,
    "hatchback": 4,
    "jeep": 5,
    "light truck": 6,
    "minibus": 7,
    "minivan": 8,
    "pickup": 9,
    "prime mover": 10,
    "sedan": 11,
    "tanker": 12,
    "truck": 13,
    "van": 14 })

#test_data.to_csv('/content/gdrive/My Drive/new_test')
#print(test_data)
new_train_cars = pd.concat([img_name, target], axis = 1, join_axes = [img_name.index])
new_train_cars.to_csv(path_or_buf = '/content/gdrive/My Drive/train_cars.csv', index = False)  # change file path
#if need index 
#new_train_cars.to_csv(path_or_buf='/content/gdrive/My Drive/train_cars.csv',index=True)
#bus: 0
#cement mixer:1 
#crane truck: 2
#dedicated agricultural vehicle
#hatchback: 4
#jeep: 5
#minibus: 7
#minivan: 8
#pickup: 9
#prime mover: 10
#sedan: 11
#tanker: 12
#truck: 13
#van:14<jupyter_output><empty_output><jupyter_text>split train and test **new csv file after mapping when spliting train and test**<jupyter_code>#split train and test 
from sklearn.model_selection import train_test_split

#load the csv file path after mapping 
train_cars=pd.read_csv('/content/gdrive/My Drive/train_cars.csv')
image_name=train_cars['image_name']
target=train_cars['target']

X_train, X_test, y_train, y_test = train_test_split(image_name, target, test_size=0.05, stratify=target)  # test size =0.05
trainset = pd.concat([X_train, y_train], axis=1, join_axes=[X_train.index])
testset = pd.concat([X_test, y_test], axis=1, join_axes=[X_test.index])
trainset.to_csv(path_or_buf='content/gdrive/My Drive/trainset.csv',index=False)   # trainset,save into my drive 
testset.to_csv(path_or_buf='content/gdrive/My Drive/testset.csv',index=False)  #test set,save into my drive
#trainset.to_csv(path_or_buf='content/gdrive/My Drive/trainset.csv',index=True)
#testset.to_csv(path_or_buf='content/gdrive/My Drive/testset.csv',index=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12771503040834056, 'fraction_numerical': 0.02403706921517521, 'mean_word_length': 6.574561403508772, 'pattern_counts': {'":': 15, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '245525', 'license': 'MIT', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1289, 'n_words': 257, 'path': 'Preprocess.ipynb', 'repo_name': 'yinino/beifen', 'size': '3453'}