<jupyter_start><jupyter_text>Expectation = Mean of Probability Mass FunctionThe probability mass function (PMF) of a random variable X is afunction which specifies the probability of obtaining a number X(ξ) = x. We denote a PMF as p_x(x) = P[X = x]<jupyter_code>import numpy as np
import matplotlib.pyplot as plt

def encrypt(t):
    chars = list(t)
    allowed_characters = list("abcdefghijklmnopqrstuvwxyz")
    mapped = []
    for char in chars:
        if char in allowed_characters:      
            mapped.append(allowed_characters.index(char))
    return mapped

def alpha_freqs(unique_elements, counts, n_alpha = 26):
    frequencies = np.zeros(26)
    for i in unique_elements:
        frequencies[i] = counts[np.where(unique_elements == i)]
    return frequencies

with open('./files/lorem_ipsum.txt', 'r') as file:
    data = file.read().replace('\n', '').lower()
    

data = np.array(encrypt(data))
unique_elements, counts = np.unique(data, return_counts=True)
f = alpha_freqs(unique_elements, counts)
n = np.arange(len('abcdefghijklmnopqrstuvwxyz'))
plt.bar(n, f/len(data))
ntag = ['a','b','c','d','e','f','g','h','i','j','k','l','m',
'n','o','p','q','r','s','t','u','v','w','x','y','z']
plt.xticks(n, ntag)
plt.show()
print(np.sum(f/len(data)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17209302325581396, 'fraction_numerical': 0.0031007751937984496, 'mean_word_length': 4.763392857142857, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '590075', 'license': 'MIT', 'n_tokens_mistral': 475, 'n_tokens_neox': 467, 'n_words': 104, 'path': 'Chapter 3.ipynb', 'repo_name': 'efeertugrul/probability4datascience', 'size': '1290'}