<jupyter_start><jupyter_text>Dullness<jupyter_code>def color_analysis(img):
#    img = Image.open(images_path+'/'+img)
    
    # obtain the color palatte of the image 
    palatte = defaultdict(int)
    for pixel in img.getdata():
        palatte[pixel] += 1
    
    # sort the colors present in the image 
    sorted_x = sorted(palatte.items(), key=operator.itemgetter(1), reverse = True)
    light_shade, dark_shade, shade_count, pixel_limit = 0, 0, 0, 25
    for i, x in enumerate(sorted_x[:pixel_limit]):
        if all(xx <= 20 for xx in x[0][:3]): ## dull : too much darkness 
            dark_shade += x[1]
        if all(xx >= 240 for xx in x[0][:3]): ## bright : too much whiteness 
            light_shade += x[1]
        shade_count += x[1]
        
    light_percent = round((float(light_shade)/shade_count)*100, 2)
    dark_percent = round((float(dark_shade)/shade_count)*100, 2)
    return light_percent, dark_percent

def perform_color_analysis(im, flag):
#    im = Image.open(images_path+'/'+im)
    
    # cut the images into two halves as complete average may give bias results
    size = im.size
    halves = (size[0]/2, size[1]/2)
    im1 = im.crop((0, 0, size[0], halves[1]))
    im2 = im.crop((0, halves[1], size[0], size[1]))

    try:
        light_percent1, dark_percent1 = color_analysis(im1)
        light_percent2, dark_percent2 = color_analysis(im2)
    except Exception as e:
        return None
    
    light_percent = (light_percent1 + light_percent2)/2 
    dark_percent = (dark_percent1 + dark_percent2)/2 
    
    if flag == 'black':
        return dark_percent
    elif flag == 'white':
        return light_percent
    else:
        return None<jupyter_output><empty_output><jupyter_text>Uniformness<jupyter_code>def average_pixel_width(im):
    im_array = np.asarray(im.convert(mode='L'))
    edges_sigma1 = feature.canny(im_array, sigma=3)
    apw = (float(np.sum(edges_sigma1)) / (im.size[0]*im.size[1]))
    return apw*100
def get_dominant_color(img):
    img = np.float32(img)
    pixels = img.reshape((-1, 3))

    n_colors = 5
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 200, .1)
    flags = cv2.KMEANS_RANDOM_CENTERS
    _, labels, centroids = cv2.kmeans(pixels, n_colors, None, criteria, 10, flags)

    palette = np.uint8(centroids)
    quantized = palette[labels.flatten()]
    quantized = quantized.reshape(img.shape)

    dominant_color = palette[np.argmax(np.unique(labels)[1])]
    return dominant_color
def get_average_color(img): 
    img = np.float32(img)
    average_color = [img[:, :, i].mean() for i in range(img.shape[-1])]
    return average_color
def getSize(filename):
    filename = images_path + '/' + filename
    st = os.stat(filename)
    return st.st_size

def getDimensions(image):
    return image.size
def get_blurrness(image):
    image = np.float32(image)
    image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    fm = cv2.Laplacian(image, cv2.CV_32F).var()
    return fm
def score_feats(data):
    data['dullness'] = data['img_mat'].apply(lambda x : perform_color_analysis(x, 'black'))
    data['whiteness'] = data['img_mat'].apply(lambda x : perform_color_analysis(x, 'white'))   
    data['apw'] = data['img_mat'].apply(lambda x : average_pixel_width(x))   
#     data['dominant_color'] = data['img_mat'].apply(lambda x : get_dominant_color(x))
    data['average_color'] = data['img_mat'].apply(lambda x : get_average_color(x))   
    data['size'] = data['image'].apply(getSize)   
    data['dim'] = data['img_mat'].apply(getDimensions)      
    data['blurrness'] = data['img_mat'].apply(lambda x: get_blurrness(x))   
    return data
start = time.time()
feats = parallelize_dataframe(feats, score_feats)
print("Feature creation time: %0.2f Minutes"%((time.time() - start)/60))
gc.collect()
feats.head()
feats.describe()
img_df = feats.drop(['img_mat'], axis= 1)
img_df.to_csv('img_df_0')
del img_df
gc.collect()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14169835234474018, 'fraction_numerical': 0.024334600760456272, 'mean_word_length': 3.6920332936979787, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '291043', 'license': 'MIT', 'n_tokens_mistral': 1476, 'n_tokens_neox': 1413, 'n_words': 285, 'path': 'notebook/Image_processing.ipynb', 'repo_name': 'wangz19/Avito_Demand_Prediction_Challenge', 'size': '3945'}