<jupyter_start><jupyter_text>Read dataset csv file and drop useless features<jupyter_code>data = pd.read_csv(CSV_PATH)
print('Data shape: {}'.format(data.shape))
data.head()
data = data[['ID', 'Doors', 'Color', 'Category']]
data.head()<jupyter_output><empty_output><jupyter_text>Keep test data separately<jupyter_code>print('Data size: {}'.format(len(data)))
data[60000:].to_csv('test_data.csv', index=False)
data = data[:60000]
print('Train data size: {}'.format(len(data)))<jupyter_output>Data size: 80577
Train data size: 60000<jupyter_text>Get car images for each car IDfor each ID get car containing image numbers, alongside drop IDs that don't have images at all<jupyter_code>from preprocess.detect_car import car_image
from preprocess import generate 
car_images = generate.get_car_images(data)
car_images.head()<jupyter_output><empty_output><jupyter_text>Merge car features and car images<jupyter_code>data = generate.merge_car_features_and_car_images(data, car_images)
print('Train data size: {}'.format(len(data)))
data.head()<jupyter_output>Train data size: 58936<jupyter_text>Generate image oriented data for model training<jupyter_code>data = generate.generate_image_oriented_dataset(data)
print('Train data size: {}'.format(len(data)))
data.head()<jupyter_output>Train data size: 533516<jupyter_text>Shuffle the dataset<jupyter_code>data = data.sample(frac=1)
data.head()<jupyter_output><empty_output><jupyter_text>Keep only the car images for our tasks<jupyter_code>data = data[data.is_car == '1']
data.drop(['is_car'], axis=1, inplace=True)
print('Train data size: {}'.format(len(data)))
data.head()<jupyter_output>Train data size: 213575<jupyter_text>Generate data for category predictionRemove useless columns<jupyter_code>category_data = data.drop(['Doors', 'Color'], axis=1)
category_data.head()
for col in category_data.Category.unique():
    length = len(category_data[category_data.Category == col])
    print('{}: {}, percentage: {}'.format(col, length, int(100*length/len(category_data))))<jupyter_output>Jeep: 77720, percentage: 36
Sedan: 93394, percentage: 43
Hatchback: 27827, percentage: 13
Goods wagon: 946, percentage: 0
Minivan: 4039, percentage: 1
Coupe: 6047, percentage: 2
Universal: 2067, percentage: 0
Microbus: 685, percentage: 0
Pickup: 342, percentage: 0
Cabriolet: 464, percentage: 0
Limousine: 44, percentage: 0<jupyter_text>Unite sets for Minivan/Microbus, Hatchback/Universal, Coupe/Cabriolet and drop Goods wagon, Pickup and Limousine as their image samples are to small and they aren't related to any of other categories<jupyter_code>def unite_categories(a):
    if a in ['Minivan', 'Microbus']:
        return 'Microbus'
    if a in ['Hatchback', 'Universal']:
        return 'Hatchback'
    if a in ['Coupe', 'Cabriolet']:
        return 'Coupe'
    if a in ['Limousine', 'Pickup', 'Goods wagon']:
        return 0
    return a

category_data.Category = category_data.Category.apply(unite_categories)
category_data = category_data[category_data.Category != 0]
for col in category_data.Category.unique():
    length = len(category_data[category_data.Category == col])
    print('{}: {}, percentage: {}'.format(col, length, int(100*length/len(category_data))))<jupyter_output>Jeep: 77720, percentage: 36
Sedan: 93394, percentage: 44
Hatchback: 29894, percentage: 14
Microbus: 4724, percentage: 2
Coupe: 6511, percentage: 3<jupyter_text>Select balanced dataset for training<jupyter_code>cat_train = category_data[category_data.Category == 'Microbus']

cat_train = pd.concat([cat_train, category_data[category_data.Category == 'Coupe']])

cat_train = pd.concat([cat_train, category_data[category_data.Category == 'Hatchback'][:10000]])

cat_train = pd.concat([cat_train, category_data[category_data.Category == 'Sedan'][:10000]])

cat_train = pd.concat([cat_train, category_data[category_data.Category == 'Jeep'][:10000]])

# Shuffle dataset
cat_train = cat_train.sample(frac=1)
for col in cat_train.Category.unique():
    length = len(cat_train[cat_train.Category == col])
    print('{}: {}, percentage: {}%'.format(col, length, int(100*length/len(cat_train))))<jupyter_output>Hatchback: 10000, percentage: 24%
Microbus: 4724, percentage: 11%
Jeep: 10000, percentage: 24%
Coupe: 6511, percentage: 15%
Sedan: 10000, percentage: 24%<jupyter_text>Generate folder containing training images<jupyter_code>cat_train = generate.generate_training_folder(cat_train, 'category_train_41K')
cat_train = cat_train[cat_train.success]
cat_train.drop(['success'], axis=1, inplace=True)
print('lenght: {}'.format(len(cat_train)))
cat_train.head()<jupyter_output>lenght: 41098<jupyter_text>Save dataframe to .csv file<jupyter_code>cat_train.to_csv('category_train_41K.csv', index=False)<jupyter_output><empty_output><jupyter_text>Generate data for door number predictionRemove useless columns<jupyter_code>door_data = data.drop(['Category', 'Color'], axis=1)
door_data.head()
for col in door_data.Doors.unique():
    length = len(door_data[door_data.Doors == col])
    print('{}: {}, percentage: {}'.format(col, length, int(100*length/len(door_data))))<jupyter_output>4/5: 202642, percentage: 94
2/3: 6725, percentage: 3
nan: 0, percentage: 0
>5: 745, percentage: 0<jupyter_text>Drop '>5' and nan values<jupyter_code>door_data.dropna(inplace=True)
door_data = door_data[door_data.Doors != '>5']
print('length: {}'.format(len(door_data)))
door_data.head()<jupyter_output>length: 209367<jupyter_text>Create balanced dataset for trianing<jupyter_code>door_train = pd.concat([door_data[door_data.Doors == '2/3'], door_data[door_data.Doors == '4/5'][:20000]])
# Shuffle
door_train = door_train.sample(frac=1)
for col in door_train.Doors.unique():
    length = len(door_train[door_train.Doors == col])
    print('{}: {}, percentage: {}%'.format(col, length, int(100*length/len(door_train))))<jupyter_output>2/3: 6725, percentage: 25%
4/5: 20000, percentage: 74%<jupyter_text>Generate folder containing training images<jupyter_code>door_train = generate.generate_training_folder(door_train, 'door_train_26K')
door_train = door_train[door_train.success]
door_train.drop(['success'], axis=1, inplace=True)
print('lenght: {}'.format(len(door_train)))
door_train.head()<jupyter_output>lenght: 26642<jupyter_text>Save dataframe to csv<jupyter_code>door_train.to_csv('door_train_26K.csv', index=False)<jupyter_output><empty_output><jupyter_text>Generate data for car color predictionRemove useless columns<jupyter_code>color_data = data.drop(['Category', 'Doors'], axis=1)
color_data.dropna(inplace=True)
print('lenght: {}'.format(len(color_data)))
color_data.head()
for col in color_data.Color.unique():
    length = len(color_data[color_data.Color == col])
    print('{}: {}, percentage: {}'.format(col, length, int(100*length/len(color_data))))<jupyter_output>Blue: 16164, percentage: 7
Grey: 29172, percentage: 13
White: 49207, percentage: 23
Black: 56194, percentage: 26
Silver: 38742, percentage: 18
Green: 3421, percentage: 1
Yellow: 692, percentage: 0
Golden: 1032, percentage: 0
Orange: 1078, percentage: 0
Purple: 231, percentage: 0
Red: 7274, percentage: 3
Brown: 2502, percentage: 1
Sky blue: 972, percentage: 0
Carnelian red: 1401, percentage: 0
Pink: 157, percentage: 0
Beige: 1333, percentage: 0<jupyter_text>Create balanced training data<jupyter_code>black = color_data[color_data.Color == 'Black'][:5000]
blue = color_data[color_data.Color == 'Blue'][:5000]
white = color_data[color_data.Color == 'White'][:5000]
silver = color_data[color_data.Color == 'Silver'][:5000]
grey = color_data[color_data.Color == 'Grey'][:5000]
green = color_data[color_data.Color == 'Green']
red = color_data[color_data.Color == 'Red'][:5000]

color_train = pd.concat([black, blue, white, silver, grey, green, red])
# Shuffle
color_train = color_train.sample(frac=1)
print('Length: {}'.format(len(color_train)))
for col in color_train.Color.unique():
    length = len(color_train[color_train.Color == col])
    print('{}: {}, percentage: {}%'.format(col, length, int(100*length/len(color_train))))
color_train = generate.generate_training_folder(color_train, 'color_train_33K')
color_train = color_train[color_train.success]
color_train.drop(['success'], axis=1, inplace=True)
print('lenght: {}'.format(len(color_train)))
color_train.head()<jupyter_output>lenght: 33326<jupyter_text>Save dataframe to csv file<jupyter_code>color_train.to_csv('color_train_33K.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14518236901508852, 'fraction_numerical': 0.047522870381371034, 'mean_word_length': 7.936305732484076, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 70, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '199316', 'license': 'MIT', 'n_tokens_mistral': 3256, 'n_tokens_neox': 2921, 'n_words': 618, 'path': 'FLOW.ipynb', 'repo_name': 'bnMikheili/Car-Feature-Detection', 'size': '8417'}