<commit_before>"""
file: ti_encode.py
language: python3
author: <NAME> <<EMAIL>>
description: Compiles plaintext to TI-Basic files and saves the file.

TODO:
    complete overhaul of how the file is read so that comments and
    text stop being a problem.
    
    Syntax checking.  Dream big, right?  Few basics would be nice, shouln't
    be too hard
    
    validation to make sure the file put in is actually a text file.  Doesn't
    break things if it isn't but it looks nasty
    
"""
import binascii
from copy import deepcopy
import dictionaries
import tiFile
import common
from metadata import createMeta


def init():    
    print("TI BASIC file encoder.  Compiles text to TIBASIC.\n")
    print("<NAME> <<EMAIL>>.")
    print("Visit www.thenaterhood.com/projects for more software\n")
    
def readFile(filename):
    """
    Reads a file into an array as chunks separated by spaces.
    
    Arguments:
        filename (string): the name of the file to open
    Returns:
        fileContents (list): byte values from the file
        
    """
    fileContents = []
    
    # Opens the file and reads it one word at a time into an array
    for line in open(filename, "r"):
        fileContents.append(line)
        fileContents.append(b'?')
    return fileContents

    
def translate(dictionary, content, escapeCharExists, escapeChar):
    """
    Takes a dictionary and a list of items (such as bytes in a file)
    and checks each byte to see if it's a key in the dictionary.  If it
    is, it replaces the item with the key's value.  Supports having
    preceding escape characters.
    
    Arguments:
        dictionary (dict): a dictionary mapping byte values to values
        content (list): a list containing byte values
        escapeCharExists (boolean): whether or not to look for an escape
            character when replacing values
        escapeChar (string): an escape character to look for. Has no
            effect if escapeCharExists is valse
        
    Returns:
        translation (list): a copy of the content list with every
            recognized value replaced
    """
    
    # Make a copy of the input so the original isn't modified
    #translation = deepcopy(content)
    translation = []

    # Iterate through each index of the copy of the content
    for item in content:
        # If there is no escape character set, check if the item
        # exists in the dictionary and replace it with its value if it does
        if ( not escapeCharExists) and (item in dictionary):
            translation.append(dictionary[item])
                
        # If an escape character is set and the character at the current
        # iteration was found in the dictionary, switch out the item for
        # its compiled byte value and insert the escape character into
        # the preceding index.
        if (escapeCharExists and item in dictionary):
            translation.append(dictionary[item])
            translation.insert(-1, escapeChar)
                    
        elif (item not in dictionary):
            translation.append(item)
                
    return translation

def parseASCII(fileContents):
    """
    Parses the TI83 hex codes for standard ascii characters and replaces
    them with the ascii equivalent.
    
    Arguments:
        fileContents (list): a list containing byte values read from
            a file
    Returns:
        a list containing the file with ascii characters in the place
        of byte values that were interpreted and converted
    """
    # make a deepcopy of the list so the original isn't modified
    ascii_parsed = deepcopy(fileContents)
    
    # Grab the dictionary with values for ascii uppercase and numbers
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_dict = dictionaries.standardASCII(True)
    
    # Call translate with the uppercase dictionary, the file contents
    # and no escape code set.
    ascii_parsed = translate(ascii_dict, ascii_parsed, False, '')
      
    # Grab the dictionary with values for the lowercase letters
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_lower_dict = dictionaries.lowercaseASCII(True)    
    
    # Call translate with the lowercase dictionary, the file contents,
    # and the escape character b'\xbb' set
    ascii_parsed = translate(ascii_lower_dict, ascii_parsed, True, b'\xbb')
    
    # Grab the dictionary mapping symbols to their plaintext values and
    # reverse the key/values since the dictionary is written for decompiling.    
    ascii_symbol_dict = dictionaries.symbolsASCII(True)
    
    # Call translate with the ascii symbol dictionary, the file contents,
    # and no escape character set
    ascii_parsed = translate(ascii_symbol_dict, ascii_parsed, False, '')
    
    return ascii_parsed

def parseWhitespace(fileContents):
    """
    Parses the TI83 whitespace codes
    
    Arguments:
        fileContents (list): a list of the byte values read from the file
    Returns:
        a list of byte values with the whitespace codes replaced with
        their ascii equivalents
    """
    
    parsedFile = []
    """
    try:
        for i in range(0, len(fileContents)):
            if (fileContents[i][0] == ":"):
                parsedFile.append(b'?')
                parsedFile.append(fileContents[i][1:])
                            
            if (fileContents[i][0] != ":"):
                parsedFile.append(fileContents[i])
    except:
        pass
    """
    # Commented, because doing this doesn't actually hold any benefit
    # with how the file is currently dealt with
    
    whitespace_dict = dictionaries.whitespace(True)
    return translate(whitespace_dict, fileContents, False, '')
    #return parsedFile
    
def parseLine(line):
    """
    Parses a line of TI-Basic code from plaintext -> tiBasic.
    Observes text and comments, so such things are now safe to compile.
    Runs through and parses comments and text, followed by newlines,
    followed by TI-Basic tokens.
    
    Arguments:
        line (str): a line of plaintext tibasic code
    Returns:
        parsedLine (list): a list of bytes from the parsed line
    """
    unParsed = deepcopy(line)[1:]
    
    # Parse any comments in the line, denoted by '"', as 
    # they should be in tibasic.  This section works as expected.

    if (len(unParsed) == 0):
        return ''
        
    if (unParsed[0] == '"'):
        parsed = parseASCII(unParsed)
        return parseWhitespace(parsed)
        
    """   <commit_msg>support text output (whitespace no longer lost)<commit_after>"""
file: ti_encode.py
language: python3
author: <NAME> <<EMAIL>>
description: Compiles plaintext to TI-Basic files and saves the file.

TODO:
    complete overhaul of how the file is read so that comments and
    text stop being a problem.
    
    Syntax checking.  Dream big, right?  Few basics would be nice, shouln't
    be too hard
    
    validation to make sure the file put in is actually a text file.  Doesn't
    break things if it isn't but it looks nasty
    
"""
import binascii
from copy import deepcopy
import dictionaries
import tiFile
import common
from metadata import createMeta


def init():    
    print("TI BASIC file encoder.  Compiles text to TIBASIC.\n")
    print("<NAME> <<EMAIL>>.")
    print("Visit www.thenaterhood.com/projects for more software\n")
    
def readFile(filename):
    """
    Reads a file into an array as chunks separated by spaces.
    
    Arguments:
        filename (string): the name of the file to open
    Returns:
        fileContents (list): byte values from the file
        
    """
    fileContents = []
    
    # Opens the file and reads it one line at a time into an array
    for line in open(filename, "r"):
        fileContents.append(line)
        fileContents.append(b'?')
    return fileContents

    
def translate(dictionary, content, escapeCharExists, escapeChar):
    """
    Takes a dictionary and a list of items (such as bytes in a file)
    and checks each byte to see if it's a key in the dictionary.  If it
    is, it replaces the item with the key's value.  Supports having
    preceding escape characters.
    
    Arguments:
        dictionary (dict): a dictionary mapping byte values to values
        content (list): a list containing byte values
        escapeCharExists (boolean): whether or not to look for an escape
            character when replacing values
        escapeChar (string): an escape character to look for. Has no
            effect if escapeCharExists is valse
        
    Returns:
        translation (list): a copy of the content list with every
            recognized value replaced
    """
    
    # Make a copy of the input so the original isn't modified
    #translation = deepcopy(content)
    translation = []

    # Iterate through each index of the copy of the content
    for item in content:
        # If there is no escape character set, check if the item
        # exists in the dictionary and replace it with its value if it does
        if ( not escapeCharExists) and (item in dictionary):
            translation.append(dictionary[item])
                
        # If an escape character is set and the character at the current
        # iteration was found in the dictionary, switch out the item for
        # its compiled byte value and insert the escape character into
        # the preceding index.
        if (escapeCharExists and item in dictionary):
            translation.append(dictionary[item])
            translation.insert(-1, escapeChar)
                    
        elif (item not in dictionary):
            translation.append(item)
                
    return translation

def parseASCII(fileContents):
    """
    Parses the TI83 hex codes for standard ascii characters and replaces
    them with the ascii equivalent.
    
    Arguments:
        fileContents (list): a list containing byte values read from
            a file
    Returns:
        a list containing the file with ascii characters in the place
        of byte values that were interpreted and converted
    """
    # make a deepcopy of the list so the original isn't modified
    ascii_parsed = deepcopy(fileContents)
    
    # Grab the dictionary with values for ascii uppercase and numbers
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_dict = dictionaries.standardASCII(True)
    
    # Call translate with the uppercase dictionary, the file contents
    # and no escape code set.
    ascii_parsed = translate(ascii_dict, ascii_parsed, False, '')
      
    # Grab the dictionary with values for the lowercase letters
    # and reverse the key/value pairs since the dictionary is written
    # for decompiling
    ascii_lower_dict = dictionaries.lowercaseASCII(True)    
    
    # Call translate with the lowercase dictionary, the file contents,
    # and the escape character b'\xbb' set
    ascii_parsed = translate(ascii_lower_dict, ascii_parsed, True, b'\xbb')
    
    # Grab the dictionary mapping symbols to their plaintext values and
    # reverse the key/values since the dictionary is written for decompiling.    
    ascii_symbol_dict = dictionaries.symbolsASCII(True)
    
    # Call translate with the ascii symbol dictionary, the file contents,
    # and no escape character set
    ascii_parsed = translate(ascii_symbol_dict, ascii_parsed, False, '')
    
    return ascii_parsed

def parseWhitespace(fileContents):
    """
    Parses the TI83 whitespace codes
    
    Arguments:
        fileContents (list): a list of the byte values read from the file
    Returns:
        a list of byte values with the whitespace codes replaced with
        their ascii equivalents
    """
    
    parsedFile = []
    """
    try:
        for i in range(0, len(fileContents)):
            if (fileContents[i][0] == ":"):
                parsedFile.append(b'?')
                parsedFile.append(fileContents[i][1:])
                            
            if (fileContents[i][0] != ":"):
                parsedFile.append(fileContents[i])
    except:
        pass
    """
    # Commented, because doing this doesn't actually hold any benefit
    # with how the file is currently dealt with
    
    whitespace_dict = dictionaries.whitespace(True)
    return translate(whitespace_dict, fileContents, False, '')
    #return parsedFile
    
def parseLine(line):
    """
    Parses a line of TI-Basic code from plaintext -> tiBasic.
    Observes text and comments, so such things are now safe to compile.
    Runs through and parses comments and text, followed by newlines,
    followed by TI-Basic tokens.
    
    Arguments:
        line (str): a line of plaintext tibasic code
    Returns:
        parsedLine (list): a list of bytes from the parsed line
    """
    unParsed = deepcopy(line)[1:]
    
    # Parse any comments in the line, denoted by '"', as 
    # they should be in tibasic.  This section works as expected.

    if (len(unParsed) == 0):
        return ''
      
    for i in range(0, len(unParsed)):
        if (unParsed[i] == '"'):
            parsed = [ unParsed[:i] ] + parseWhitespace( parseASCII(unParsed[i:]) )
            return parsed
        
    """   
{'fraction_non_alphanumeric': 0.06553562044891524, 'fraction_numerical': 0.0019518054200135125, 'mean_word_length': 2.47107868681605, 'pattern_counts': {'":': 4, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1915478', 'n_tokens_mistral': 3564, 'n_tokens_neox': 3431, 'n_words': 1487}