<jupyter_start><jupyter_text>Read json to dataframe:<jupyter_code>json = '../resource/allDocstrings.json'
data = pd.read_json(json)
df = pd.DataFrame(data)


#We are only interessted in number of lines, container type and implementation type
distribution_df = pd.DataFrame(df, columns=['docstringContainerType', 'numberOfLines', 'docstringContainerImplementationType'] )<jupyter_output><empty_output><jupyter_text>Function for removing outliers:<jupyter_code>def remove_outlier(df_in, col_name):
    q1 = df_in[col_name].quantile(0.25)
    q3 = df_in[col_name].quantile(0.75)
    iqr = q3-q1 #Interquartile range
    fence_low = q1-1.5*iqr
    fence_high = q3+1.5*iqr
    df_out = df_in.loc[(df_in[col_name] > fence_low) & (df_in[col_name] < fence_high)]
    return df_out<jupyter_output><empty_output><jupyter_text>Length Distribution of Docstrings<jupyter_code>#df["numberOfLines"].plot()

#plt.plot(range(0, len(df["numberOfLines"])), df["numberOfLines"].sort_values(ascending=False))
#plt.xlabel("Number of Lines")
#plt.ylabel("Amount of Docstrings")
#plt.savefig("length_distribution.png")<jupyter_output><empty_output><jupyter_text>distribution of docstring length of first 95%<jupyter_code>first_95_percent = df.loc[(df.numberOfLines <= df["numberOfLines"].quantile(0.95)) & (df.numberOfLines > 0)]
first_95_percent.describe()
pd.value_counts(first_95_percent["numberOfLines"]).plot.bar()
one_liner_count = len(df.loc[df.numberOfLines == 1].index)
one_liner_count
docstrings_count = len(df.index)
docstrings_count

percentage_of_oneliner = one_liner_count/docstrings_count * 100
percentage_of_oneliner

plt.xlabel("Number of Lines")
plt.ylabel("Amount of Docstrings")
plt.savefig("length_distribution.png")<jupyter_output><empty_output><jupyter_text>Distribution according to Docstring Container<jupyter_code>#Create dataframes for every group according to docstring container

functions = df[df.docstringContainerType == 'FUNCTION']

classes = df[df.docstringContainerType == 'CLASS']

modules = df[df.docstringContainerType == 'MODULE']

#Get element count for every group

functions_count = len(functions.index)

classes_count = len(classes.index)

modules_count = len(modules.index)

functions_count
classes_count
modules_count

#Get percentage of elements of every group

functions_percentage = functions_count/docstrings_count * 100
classes_percentage = classes_count/docstrings_count * 100
modules_percentage = modules_count/docstrings_count * 100

functions_percentage
classes_percentage
modules_percentage<jupyter_output><empty_output><jupyter_text>Length Distribution according to Docstring Container<jupyter_code>#Remove one-liner docstrings

functions = functions[functions.numberOfLines > 1]
classes = classes[classes.numberOfLines > 1]
modules = modules[modules.numberOfLines > 1]

#remove Tests:

functions_without_tests = functions[functions.docstringContainerImplementationType != "TEST"]
classes_without_tests = classes[classes.docstringContainerImplementationType != "TEST"]
modules_without_tests = modules[modules.docstringContainerImplementationType != "TEST"]


#First remove the outliners:

functions_without_outliers = remove_outlier(functions_without_tests, "numberOfLines")
classes_without_outliers = remove_outlier(classes_without_tests, "numberOfLines")
modules_without_outliers = remove_outlier(modules_without_tests, "numberOfLines")


#Plot length distribution without one-liners and outliers:

pd.value_counts(functions_without_outliers["numberOfLines"]).plot.bar()

plt.xlabel("Number of Lines")
plt.ylabel("Amount of Docstrings")
plt.savefig("functions_length_distribution.png")


pd.value_counts(classes_without_outliers["numberOfLines"]).plot.bar()
pd.value_counts(modules_without_outliers["numberOfLines"]).plot.bar()


#Describe all groups, get q1, and q3:

functions_without_outliers.describe()
classes_without_outliers.describe()
modules_without_outliers.describe()

df = df[df.numberOfLines > 1]
df_without_tests = df[df.docstringContainerImplementationType != "TEST"]
clean_df = remove_outlier(df_without_tests, "numberOfLines")
clean_df.describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10831112188030045, 'fraction_numerical': 0.011630724497213472, 'mean_word_length': 8.781990521327014, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '576880', 'license': 'Apache-2.0', 'n_tokens_mistral': 1406, 'n_tokens_neox': 1327, 'n_words': 227, 'path': 'data-analysis/docstring-length-and-component-distribution/distribution-analysis/LengthAndComponentDistribution.ipynb', 'repo_name': 'mjakov95/bachelorThesis', 'size': '4127'}