<jupyter_start><jupyter_text>Entropy is the measure of "average surprisal" per unit of information transmitted. (Or, in the case of storage, stored.)<jupyter_code>from math import log
READ_FROM_FILE = False
if READ_FROM_FILE:
    with open(input("Please enter the filename: ")) as f:
        text = f.read()
else:
    text = input("Please enter the text, and press enter when you are finished.\n")
ALPHANUMERICS_ONLY = False
# letter by letter
if ALPHANUMERICS_ONLY:
    text = ''.join([letter for letter in text if letter.isalnum()])
set_of_letters = set(text)
letters_entropy = 0.0
for letter in set_of_letters:
    freq = text.count(letter)/len(text)
    letters_entropy += - freq*log(freq)

theoretical_max_entropy = -log(1/len(set_of_letters)) # maximum entropy you can achieve using only this set of letters.
print("The theoretical maximum entropy for this given length of string using this set of characters =",
      theoretical_max_entropy*len(text), "nats, at ", theoretical_max_entropy, "nats per character.")

print("The actual amount of entropy this message contains is ", letters_entropy*len(text), "nats,")
print("which is equivalent to ", letters_entropy/log(2)*len(text), "bits.")
print("The entropy per letter = ", letters_entropy, "nats = ", letters_entropy/log(2), "bits.")
print("Therefore transmitting the message character by character will take", letters_entropy/log(2)*len(text),
      "bits even after compressing it optimally.")

print("A compression algorithm optimized to transmit this specific string can reduce the size of this string by",
      (1-(letters_entropy/theoretical_max_entropy))*100, "% down to its minimum value stated above.")
# word by word
words = text.split()
set_of_words = set(words)
words_entropy = 0.0
for word in set_of_words:
    instance = len([match for match in text.split() if match==word])
    freq = instance/len(text)
    words_entropy += - freq*log(freq)

theoretical_max_entropy = -log(1/len(set_of_words)) # maximum entropy you can achieve using only this set of words.
print("The theoretical maximum entropy for this given length of string using this set of characters =",
      theoretical_max_entropy*len(text), "nats, at ", theoretical_max_entropy, "nats per character.")

print("The actual amount of entropy this message contains is ", words_entropy*len(text), "nats,")
print("which is equivalent to ", words_entropy/log(2)*len(text), "bits.")
print("The entropy per word = ", words_entropy, "nats = ", words_entropy/log(2), "bits.")
print("Therefore transmitting the message character by character will take", words_entropy/log(2)*len(text),
      "bits even after compressing it optimally.")

print("A compression algorithm optimized to transmit this specific string can reduce the size of this string by",
      (1-(words_entropy/theoretical_max_entropy))*100, "% down to its minimum value stated above.")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1076341127922971, 'fraction_numerical': 0.0068775790921595595, 'mean_word_length': 5.215811965811966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '526671', 'license': 'MIT', 'n_tokens_mistral': 874, 'n_tokens_neox': 848, 'n_words': 302, 'path': 'text_entropy.ipynb', 'repo_name': 'OceanNuclear/Entropy', 'size': '2908'}