<jupyter_start><jupyter_text>Load JSON dataset by pandas<jupyter_code>ratings_grade_inflation_dataframe = pd.read_json('../data/ratings_grade_inflation.json')
ratings_grade_inflation_dict = ratings_grade_inflation_dataframe.to_dict()<jupyter_output><empty_output><jupyter_text>Display dataset<jupyter_code>ratings_grade_inflation_dataframe = ratings_grade_inflation_dataframe.transpose()
display(ratings_grade_inflation_dataframe)
PlotHelper(ratings_grade_inflation_dict,'saeed-alireza').draw_username('saeed','alireza')
PlotHelper(ratings_grade_inflation_dict,'saeed-abbas').draw_username('saeed','abbas')<jupyter_output><empty_output><jupyter_text>implementation<jupyter_code>def get_manhattan_distance(rating1,rating2):
    
    rating1 = dict(filter(lambda x: x[1] != '-', rating1.items()))
    rating2 = dict(filter(lambda x: x[1] != '-', rating2.items()))
    
    distance = 0
    for key in rating1:
        if key in rating2:
            distance += abs(rating1[key] - rating2[key])
    return distance

dt=ratings_grade_inflation_dict

print('saeed-alireza:%f'%get_manhattan_distance(dt['saeed'],dt['alireza']))
print('saeed-abbas:%f'%+get_manhattan_distance(dt['saeed'],dt['abbas']))
def get_pearson(rating1, rating2):
    sum_xy = 0
    sum_x = 0
    sum_y = 0
    sum_x2 = 0
    sum_y2 = 0
    n = 0
    for key in rating1:
        if key in rating2:
            n += 1
            x = rating1[key]
            y = rating2[key]
            sum_xy += x * y
            sum_x += x
            sum_y += y
            sum_x2 += pow(x, 2)
            sum_y2 += pow(y, 2)
    denominator = sqrt(sum_x2 - pow(sum_x, 2) / n) * sqrt(sum_y2 - pow(sum_y, 2) / n)
    if denominator == 0:
        return 0
    else:
        return (sum_xy - (sum_x * sum_y) / n) / denominator

dt=ratings_grade_inflation_dict

print('saeed-alireza:%f'%get_pearson(dt['saeed'],dt['alireza']))
print('saeed-abbas:%f'%+get_pearson(dt['saeed'],dt['abbas']))<jupyter_output>saeed-alireza:0.982708
saeed-abbas:-0.316228
{'fraction_non_alphanumeric': 0.13770655983975963, 'fraction_numerical': 0.026039058587881823, 'mean_word_length': 4.071065989847716, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '549162', 'license': 'MIT', 'n_tokens_mistral': 780, 'n_tokens_neox': 762, 'n_words': 109, 'path': 'datamining_python/examples/pearson_correlation_coefficient.ipynb', 'repo_name': 'mberneti/datamining-python', 'size': '1997'}