<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = "<NAME>"
__license__ = "BSD-new"

"""
Use a probabilistic letter model to 'break' a rotation cipher.

This solution uses a very simple model (but it's enough to decode the message):
    * 2 letter bigrams built from a list of words (around 260,000 words).
    * Naïve Bayes assumption: P(bg_1, bg_2... bg_n) = Product(i:1..n) P(bg_i).

Requirements:
    * Python 3.x
"""

import functools
import logging
import os.path
import pickle
import re
import time

# Logging level
logging.basicConfig(level=logging.INFO)


# Text we're trying to decode
TEXT = "Esp qtcde nzyqpcpynp zy esp ezatn zq Lcetqtntlw \
Tyepwwtrpynp hld spwo le Olcexzfes Nzwwprp ty estd jplc."

ALPHABET_EN = [
    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
    "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
]


class RotationCipher:
    """Rotates character strings"""

    def __init__(self, alphabet=ALPHABET_EN, valid_chars="[a-z]"):
        self.alphabet = alphabet
        self.valid_chars = re.compile(valid_chars)

    def rotate_char(self, char, shift=0):
        """Map from one character to another by a shift of size N"""
        if self.valid_chars.match(char):
            char = self.alphabet[(self.alphabet.index(char) + shift) % len(self.alphabet)]
        return char

    def encode(self, text="", shift=0):
        """Map a string to another by a shift of size N"""
        return "".join([self.rotate_char(c, shift) for c in text.lower()])


class LetterBigrams:
    """Create letter bigrams from a word list"""

    def __init__(self, alphabet=ALPHABET_EN):
        self.alphabet = alphabet
        self.__words_file = os.path.join(os.path.dirname(__file__), "sowpods.txt")
        self.__words_p = os.path.join(os.path.dirname(__file__), ".words.p")
        self.__prob_model_p = os.path.join(os.path.dirname(__file__), ".prob_model.p")

        if os.path.isfile(self.__words_p):
            self.words = pickle.load(open(self.__words_p, "rb"))<commit_msg>Reuse the current directory location<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
__author__ = "<NAME>"
__license__ = "BSD-new"

"""
Use a probabilistic letter model to 'break' a rotation cipher.

This solution uses a very simple model (but it's enough to decode the message):
    * 2 letter bigrams built from a list of words (around 260,000 words).
    * Naïve Bayes assumption: P(bg_1, bg_2... bg_n) = Product(i:1..n) P(bg_i).

Requirements:
    * Python 3.x
"""

import functools
import logging
import os.path
import pickle
import re
import time

# Logging level
logging.basicConfig(level=logging.INFO)


# Text we're trying to decode
TEXT = "Esp qtcde nzyqpcpynp zy esp ezatn zq Lcetqtntlw \
Tyepwwtrpynp hld spwo le Olcexzfes Nzwwprp ty estd jplc."

ALPHABET_EN = [
    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m",
    "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"
]


class RotationCipher:
    """Rotates character strings"""

    def __init__(self, alphabet=ALPHABET_EN, valid_chars="[a-z]"):
        self.alphabet = alphabet
        self.valid_chars = re.compile(valid_chars)

    def rotate_char(self, char, shift=0):
        """Map from one character to another by a shift of size N"""
        if self.valid_chars.match(char):
            char = self.alphabet[(self.alphabet.index(char) + shift) % len(self.alphabet)]
        return char

    def encode(self, text="", shift=0):
        """Map a string to another by a shift of size N"""
        return "".join([self.rotate_char(c, shift) for c in text.lower()])


class LetterBigrams:
    """Create letter bigrams from a word list"""

    def __init__(self, alphabet=ALPHABET_EN):
        self.alphabet = alphabet

        cwd = os.path.dirname(__file__)
        self.__words_file = os.path.join(cwd, "sowpods.txt")
        self.__words_p = os.path.join(cwd, ".words.p")
        self.__prob_model_p = os.path.join(cwd, ".prob_letter_model.p")

        if os.path.isfile(self.__words_p):
            self.words = pickle.load(open(self.__words_p, "rb"))
{'fraction_non_alphanumeric': 0.15345454545454545, 'fraction_numerical': 0.006787878787878788, 'mean_word_length': 3.257997936016512, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4410150', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1462, 'n_words': 419}