<commit_before>import os
import sys
import re
import pymongo as pm


def open_asm(filename):
    # datapath = '/Users/carlyhendrickson/git/microsoft_malware_challenge/data/train/' + filename + '.asm'
    datapath = '/nfs/research/agonzales/git/microsoft_malware_challenge/data/train/' + filename + '.asm'
    with open(datapath, 'rb') as f:
        asm = [line for line in f.readlines()]
        return asm


def to_utf(line):
    utf_line = ''
    try:
        utf_line = str(line, 'utf8')
        return utf_line
    except UnicodeDecodeError as u:
        # print(u)
        return None


def filter_comments(asm_utf):
    """Filters over an assembly file and gives the IDA pro comments
    Requires a UTF8 set of strings."""
    comments = []
    # removes nones<commit_msg>added comments<commit_after>"""Program to pull the comments out of an ida-dissasembled program"""

__author__ = "<NAME>, <NAME>"
__license__ = "Apache"
__email__ = "<EMAIL>"


import sys
import re
import pymongo as pm


def open_asm(filename):
    """Reads all the lines in an assembly file correctly.
    Args: 
        filename (str): the hash name of the file. e.g., "haa4945azkda09", no
                        path or extension
        returns:
            a list of all the decoded strings in bytecode format
    """
    # datapath = '/Users/carlyhendrickson/git/microsoft_malware_challenge/data/train/' + filename + '.asm'
    datapath = '/nfs/research/agonzales/git/microsoft_malware_challenge/data/train/' + filename + '.asm'
    with open(datapath, 'rb') as f:
        asm = [line for line in f.readlines()]
        return asm


def to_utf(line):
    """Converts a bytecode string to unicode. Ignores lines that have unicode
    decode errors.
    Args:
        line (string) a single line in the file
    Returns:
        a single line that's been cleaned
    """
    utf_line = ''
    try:
        utf_line = str(line, 'utf8')
        return utf_line
    except UnicodeDecodeError as u:
        # print(u)
        return None


def read_assembly(filename):
    """ Reads an assembly file and returns a nice UTF version of it.
    Args:
        filename (str) root of the file's filename, no extension
    Returns:
        list of all lines in the file
    """
    my_asm = open_asm(filename)
    asm = [to_utf(line) for line in my_asm]
    return asm


def filter_comments(asm_utf):
    """Filters over an assembly file and gives the IDA pro comments
    Requires a UTF8 set of strings."""
    comments = []
    # removes nones
{'fraction_non_alphanumeric': 0.09574890742947954, 'fraction_numerical': 0.003972983710766786, 'mean_word_length': 2.6705539358600583, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6191898', 'n_tokens_mistral': 804, 'n_tokens_neox': 762, 'n_words': 263}