<jupyter_start><jupyter_text>Test di normalità sulle variabili numeriche del dataset<jupyter_code>numerical_data = dataset[['version', 'tags_number', 'score', 'number_of_gif', 'maker_comment_ratio', 'thread_length']]
numerical_data.head()
def histogram_plot(data, histogram_title, figure_name):
    plt.figure(figsize=(7,6))
    plt.title(histogram_title, fontsize=14)
    plt.hist(data)
    plt.savefig(figure_name)
def qq_plot(data, figure_name):
    plt.figure(figsize=(7,6))
    qqplot(data, line='s')
    plt.savefig(figure_name)
def shapiro_test(data):
    stat, p_value = shapiro(data)
    print("Statistics = {:.3f}, p-value = {:.3f}".format(stat, p_value))
    alpha = 0.05
    if p_value > alpha:
        print('Sample looks Gaussian (fail to reject H0)')
    else:
        print('Sample does not look Gaussian (reject H0)')<jupyter_output><empty_output><jupyter_text>Variabile *version*<jupyter_code>histogram_plot(numerical_data['version'], 'Distribution of version variable', 'version_distribution_(histogram).png')
qq_plot(numerical_data['version'], 'version_distribution_(quantile-quantile).png')
shapiro_test(numerical_data['version'])<jupyter_output>Statistics = 0.270, p-value = 0.000
Sample does not look Gaussian (reject H0)<jupyter_text>Variabile *tags number*<jupyter_code>histogram_plot(numerical_data['tags_number'], 'Distribution of tags number variable', 
               'tags_number_distribution_(histogram).png')
qq_plot(numerical_data['tags_number'], 'tags_number_distribution_(quantile-quantile).png')
shapiro_test(numerical_data['tags_number'])<jupyter_output>Statistics = 0.861, p-value = 0.000
Sample does not look Gaussian (reject H0)<jupyter_text>Variabile score<jupyter_code>histogram_plot(numerical_data['score'], 'Distribution of score variable', 'score_distribution_(histogram).png')
qq_plot(numerical_data['score'], 'score_distribution_(quantile-quantile).png')
shapiro_test(numerical_data['score'])<jupyter_output>Statistics = 0.612, p-value = 0.000
Sample does not look Gaussian (reject H0)<jupyter_text>Variabile *number of gif*<jupyter_code>histogram_plot(numerical_data['number_of_gif'], 'Distribution of number of gif variable', 
               'number_of_gif_distribution_(histogram).png')
qq_plot(numerical_data['number_of_gif'], 'number_of_gif_distribution_(quantile-quantile).png')
shapiro_test(numerical_data['number_of_gif'])<jupyter_output>Statistics = 0.515, p-value = 0.000
Sample does not look Gaussian (reject H0)<jupyter_text>Variabile *maker comment ratio*<jupyter_code>histogram_plot(numerical_data['maker_comment_ratio'], 'Distribution of maker comment ratio variable', 
               'maker_comment_ratio_distribution_(histogram).png')
qq_plot(numerical_data['maker_comment_ratio'], 'maker_comment_ratio_distribution_(quantile-quantile).png')
shapiro_test(numerical_data['maker_comment_ratio'])<jupyter_output>Statistics = 0.505, p-value = 0.000
Sample does not look Gaussian (reject H0)<jupyter_text>Variabile *thread length*<jupyter_code>histogram_plot(numerical_data['thread_length'], 'Distribution of thread length variable', 
               'thread_length_distribution_(histogram).png')
qq_plot(numerical_data['thread_length'], 'thread_length_distribution_(quantile-quantile).png')
shapiro_test(numerical_data['thread_length'])<jupyter_output>Statistics = 0.615, p-value = 0.000
Sample does not look Gaussian (reject H0)
{'fraction_non_alphanumeric': 0.12396694214876033, 'fraction_numerical': 0.01977567886658796, 'mean_word_length': 7.89501312335958, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '103644', 'license': 'MIT', 'n_tokens_mistral': 1175, 'n_tokens_neox': 1102, 'n_words': 187, 'path': 'analysis/notebook/normality_test/normality_test.ipynb', 'repo_name': 'YashovardhanSingh/PH_miner', 'size': '3388'}