<commit_before>import learn
import likelihood
import sys
import random
import math
from optparse import OptionParser	#Deprecated since Python 2.7 but...

def usage():
    """Print information on how to use decode.py."""
    print "Usage: python decode.py ..."

def decode(ciphertext,targetAlphabet,numSteps,verbose=False):
    return "plaintext"
	
def parse_options():
    parser = OptionParser()
    parser.add_option("-c", "--cipher", dest="ciphertextFilename")
    parser.add_option("-a", "--alphabet", dest="alphabetFilename")
    parser.add_option("-o", "--output", dest="outputFilename")
    parser.add_option("-t", "--training", dest="trainingFilename")
    parser.add_option("-n","--numsteps",dest="numSteps")
    parser.add_option("-v", "--verbose", action="store_true",dest="verbose", default=False)
    options, args = parser.parse_args()
    if not options.outputFilename:
        print "Please specify an output file for plain text with option -o"
        sys.exit()
    if not options.ciphertextFilename:
        print "Please specify ciphertext file with option -c"
        sys.exit()
    if not options.trainingFilename:
        print "Please specify training text filename with option -t"
        sys.exit()
    if not options.alphabetFilename:
        if options.verbose:
            print "Reading target alphabet from alphabet.txt"
        alphabetFilename = "alphabet.txt"
    else:
        alphabetFilename = options.alphabetFilename
    if not options.numSteps:
        print "Please specify a number of Monte Carlo steps with option -n"
        sys.exit()
    else:
        steps = int(options.numSteps)
    return options, args

def read_texts(options):
    # Read in target alphabet.
    try:
        f = open(options.alphabetFilename,'r')
        alphabet = [c for c in f.readline().rstrip('\n')]
        f.close()
    except IOError:
        print "Error opening alphabet file " + alphabetFilename
        sys.exit()
        
    # Read in training text.  
    try:
        if options.verbose:
            print "Reading training text from " + options.trainingFilename + "..."
        f = open(options.trainingFilename)
        lines = f.readlines()
        training = "".join(lines).upper() #Make it all uppercase
        training = [c for c in training if c in alphabet] #Only include symbols included in target alphabet
        f.close()
    except IOError:
        print "Error opening training text file " + options.trainingFilename
        sys.exit()

    # Read in ciphertext.    
    try:
        f = open(options.ciphertextFilename)
        ciphertext = "".join([l.rstrip('\n') for l in f.readlines()])
        f.close()
    except IOError:
        print "Error opening ciphertext file " + options.ciphertextFilename
        sys.exit()
    return alphabet, training, ciphertext
    
def init_cipherAlphabet(alphabet, training, ciphertext):
    # Make sure target alphabet has at least as many symbols as ciphertext.
    cipherAlphabet = list(set(ciphertext))
    if len(cipherAlphabet) > len(alphabet):
        print "Error: ciphertext contains more distinct symbols than target alphabet"
        sys.exit()

    # Special case: If ciphertext alphabet is a subset of target alphabet
    # Then give them the same order
    if set(cipherAlphabet) < set(alphabet):
        cipherAlphabet = alphabet
	    
    # Sort initial guess according to letter frequencies.
    cipherFrequencies = learn.learnVector(ciphertext, cipherAlphabet)
    trainingFrequencies = learn.learnVector(training, alphabet)<commit_msg>Added docstrings for read_texts and init_cipherAlphabet.<commit_after>import learn
import likelihood
import sys
import random
import math
from optparse import OptionParser	#Deprecated since Python 2.7 but...

def usage():
    """Print information on how to use decode.py."""
    print "Usage: python decode.py ..."

def decode(ciphertext,targetAlphabet,numSteps,verbose=False):
    return "plaintext"
	
def parse_options():
    """Parse the command line options."""
    parser = OptionParser()
    parser.add_option("-c", "--cipher", dest="ciphertextFilename")
    parser.add_option("-a", "--alphabet", dest="alphabetFilename")
    parser.add_option("-o", "--output", dest="outputFilename")
    parser.add_option("-t", "--training", dest="trainingFilename")
    parser.add_option("-n","--numsteps",dest="numSteps")
    parser.add_option("-v", "--verbose", action="store_true",dest="verbose", default=False)
    options, args = parser.parse_args()
    if not options.outputFilename:
        print "Please specify an output file for plain text with option -o"
        sys.exit()
    if not options.ciphertextFilename:
        print "Please specify ciphertext file with option -c"
        sys.exit()
    if not options.trainingFilename:
        print "Please specify training text filename with option -t"
        sys.exit()
    if not options.alphabetFilename:
        if options.verbose:
            print "Reading target alphabet from alphabet.txt"
        alphabetFilename = "alphabet.txt"
    else:
        alphabetFilename = options.alphabetFilename
    if not options.numSteps:
        print "Please specify a number of Monte Carlo steps with option -n"
        sys.exit()
    else:
        steps = int(options.numSteps)
    return options, args

def read_texts(options):
    """Read in target alphabet, training text, and ciphertext."""
    # Read in target alphabet.
    try:
        f = open(options.alphabetFilename,'r')
        alphabet = [c for c in f.readline().rstrip('\n')]
        f.close()
    except IOError:
        print "Error opening alphabet file " + alphabetFilename
        sys.exit()
        
    # Read in training text.  
    try:
        if options.verbose:
            print "Reading training text from " + options.trainingFilename + "..."
        f = open(options.trainingFilename)
        lines = f.readlines()
        training = "".join(lines).upper() #Make it all uppercase
        training = [c for c in training if c in alphabet] #Only include symbols included in target alphabet
        f.close()
    except IOError:
        print "Error opening training text file " + options.trainingFilename
        sys.exit()

    # Read in ciphertext.    
    try:
        f = open(options.ciphertextFilename)
        ciphertext = "".join([l.rstrip('\n') for l in f.readlines()])
        f.close()
    except IOError:
        print "Error opening ciphertext file " + options.ciphertextFilename
        sys.exit()
    return alphabet, training, ciphertext
    
def init_cipherAlphabet(alphabet, training, ciphertext):
    """Initialize the cipher alphabet and generate the initial guess permutation."""
    # Make sure target alphabet has at least as many symbols as ciphertext.
    cipherAlphabet = list(set(ciphertext))
    if len(cipherAlphabet) > len(alphabet):
        print "Error: ciphertext contains more distinct symbols than target alphabet"
        sys.exit()

    # Special case: If ciphertext alphabet is a subset of target alphabet
    # Then give them the same order
    if set(cipherAlphabet) < set(alphabet):
        cipherAlphabet = alphabet
	    
    # Sort initial guess according to letter frequencies.
    cipherFrequencies = learn.learnVector(ciphertext, cipherAlphabet)
    trainingFrequencies = learn.learnVector(training, alphabet)
{'fraction_non_alphanumeric': 0.1016115815350997, 'fraction_numerical': 0.0005462988254575252, 'mean_word_length': 2.951969778737183, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '278930', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1943, 'n_words': 643}