<jupyter_start><jupyter_text>Section 2<jupyter_code>%run preamble.py
danish = pd.read_csv("../Data/danish.csv").x<jupyter_output><empty_output><jupyter_text>Descriptive statistics , histogram and boxplot<jupyter_code>print(danish.describe().to_latex())
fig, axs = plt.subplots(1, 2, tight_layout=True, figsize=(5, 2.5))
num_bins = 40
# the histogram of the data
n, bins, patches = axs[0].hist(danish, num_bins, alpha=0.5)
axs[1].boxplot(danish)
axs[0].set_xlabel("Loss")
axs[0].set_ylabel("Frequency")
axs[1].set_ylabel("Loss")
axs[1].set_xticks([])
# plt.show()
sns.despine()
plt.savefig("../Figures/hist_boxplot_danish_en.pdf")<jupyter_output><empty_output><jupyter_text>Maximum likelihood estimation of common loss distributions Fitting the gamma distribution<jupyter_code>danish = pd.read_csv("../Data/danish.csv").x.values
np.random.seed(333)
r, m, BIC, AIC = infer_gamma(danish, (1,1))
fit_gamma = pd.DataFrame({'shape': np.array([r]), "scale": np.array([m]), "AIC":np.array([AIC]), "BIC":np.array([BIC]) })
print(fit_gamma)
X0 = np.random.gamma(shape = r, scale  = m, size=100000)
# print(X0)
qq_plot_en(danish, X0, "blue")
sns.despine()
plt.savefig("../Figures/QQplot_gamma_en.pdf")<jupyter_output>shape    scale           AIC           BIC
0  1.257994  2.43459  10490.053765  10501.695447<jupyter_text>Fitting the lognormal distribution<jupyter_code>np.random.seed(333)
μ, σ, BIC, AIC = infer_lnorm(danish)
fit_lnorm = pd.DataFrame({'shape': np.array([σ]), "scale": np.array([μ]),"AIC":np.array([AIC]),"BIC":np.array([BIC])  })
print(fit_lnorm)
X0 = np.random.lognormal(mean = μ, sigma  = σ, size=100000)
qq_plot_en(danish, X0, "red")
sns.despine()
plt.savefig("../Figures/QQplot_lnorm_en.pdf")<jupyter_output>shape     scale          AIC          BIC
0  0.671854  0.732317  8871.781777  8883.423459<jupyter_text>Fitting the Weibull distribution<jupyter_code>np.random.seed(333)
k, β, BIC, AIC = infer_weib(danish, (1,1))
fit_weibull = pd.DataFrame({'shape': np.array([k]), "scale": np.array([β]), "AIC":np.array([AIC]), "BIC":np.array([BIC]) })
print(fit_weibull)
X0 = np.random.weibull(k, size=100000)* β
qq_plot_en(danish, X0, "green")
sns.despine()
plt.savefig("../Figures/QQplot_weib_en.pdf")<jupyter_output>shape     scale           AIC           BIC
0  0.947587  2.952484  10544.941033  10556.582715<jupyter_text>Fitting the Pareto distribution<jupyter_code>np.random.seed(333)
θ, α, BIC, AIC = infer_par(danish)
fit_pareto = pd.DataFrame({'shape': np.array([θ]), "scale": np.array([α]), "AIC":np.array([AIC]), "BIC":np.array([BIC]) })
print(fit_pareto)
X0 = θ * np.exp(np.random.exponential(size=100000 )/ α)
qq_plot_en(danish, X0, "magenta")
sns.despine()
plt.savefig("../Figures/QQplot_pareto_en.pdf")<jupyter_output>shape     scale           AIC          BIC
0  0.313404  0.545817  11354.188278  11365.82996<jupyter_text>Maximum likelihood estimation of composite models Fitting the gamma-Pareto model<jupyter_code>np.random.seed(333)
K = 100000
parms, BIC, AIC = mle_composite(danish, (1,1,1), "gam-par")
fit_gam_par = pd.DataFrame(np.append(parms, [AIC, BIC])).T
fit_gam_par.columns = ["shape", "tail", "thres", "AIC","BIC"]
print(fit_gam_par)
X0 = sim_gam_par(K, parms[0], parms[1], parms[2])
qq_plot_en(danish, X0, "blue")
sns.despine()
plt.savefig("../Figures/QQplot_gam_par_en.pdf")<jupyter_output>shape      tail     thres          AIC          BIC
0  35.667524  1.313008  1.155924  7723.681796  7741.144319<jupyter_text>Fitting the Weibull-Pareto model<jupyter_code>np.random.seed(333)
parms, BIC, AIC = mle_composite(danish, (1,1,1), "wei-par")
fit_wei_par = pd.DataFrame(np.append(parms, [AIC, BIC])).T
fit_wei_par.columns = ["shape", "tail", "thres", "AIC","BIC"]
print(fit_wei_par)
X0 = sim_wei_par(K, parms[0], parms[1], parms[2])
qq_plot_en(danish, X0, "green")
sns.despine()
plt.savefig("../Figures/QQplot_wei_par_en.pdf")<jupyter_output>shape      tail     thres         AIC          BIC
0  14.033784  1.261477  1.002991  7686.75154  7704.214063<jupyter_text>Fitting the lognormal-Pareto<jupyter_code>np.random.seed(333)
parms, BIC, AIC = mle_composite(danish, (0.5,1,1), "lnorm-par")
fit_lnorm_par = pd.DataFrame(np.append(parms, [AIC, BIC])).T
fit_lnorm_par.columns = ["shape", "tail", "thres", "AIC","BIC"]
print(fit_lnorm_par)
X0 = sim_lnorm_par(K, parms[0], parms[1], parms[2])
qq_plot_en(danish, X0, "red")
sns.despine()
plt.savefig("../Figures/QQplot_lnorm_par_en.pdf")<jupyter_output>shape      tail    thres          AIC          BIC
0  0.196517  1.328223  1.20743  7737.728264  7755.190787
{'fraction_non_alphanumeric': 0.17442116868798235, 'fraction_numerical': 0.08269018743109151, 'mean_word_length': 5.179836512261581, 'pattern_counts': {'":': 12, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '563295', 'license': 'MIT', 'n_tokens_mistral': 2199, 'n_tokens_neox': 1967, 'n_words': 307, 'path': 'Python_paper/Section2.ipynb', 'repo_name': 'LaGauffre/SMCCompoMo', 'size': '4535'}