<commit_before>import os
import sys
import json
import datetime
import argparse
import glob
try:
    from colorama import init, Fore, Back, Style
except ImportError:
    print("Error importing colorama. Please make sure you have it (pip install colorama)")
    sys.exit(-1)

try:
    from BinarlyAPIv1 import BinarlyAPI, hex_pattern, ascii_pattern, wide_pattern, build_query
except ImportError:
    print("Error importing BinarlyAPI. You can find it here https://github.com/binarlyhq/binarly-sdk")
    sys.exit(-1)

BINOBJ = None
APIKEYFILENAME = 'apikey.txt'
APIKEYPATH = os.path.join(os.path.dirname(__file__), APIKEYFILENAME)

APIKEY = ''

ARGPARSER = argparse.ArgumentParser(description='Binarly API Query')
ARGPARSER.add_argument("--key", "-k", help="Binarly APIKey", default='')
ARGPARSER.add_argument("--usehttp", "-u", help="Use HTTP instead of HTTPS when communicating. By default HTTPS is used.", action="store_true")
ARG_SUBPARSERS = ARGPARSER.add_subparsers(help='commands', dest='commands')

SEARCH_PARSER = ARG_SUBPARSERS.add_parser('search', help="Search arbitrary hex patterns")
SEARCH_PARSER.add_argument("hex", type=str, nargs='*', default=[])
SEARCH_PARSER.add_argument("-a", type=str, nargs='*', help="ASCII string to search", default=[])
SEARCH_PARSER.add_argument("-w", type=str, nargs='*', help="WIDE string to search", default=[])
SEARCH_PARSER.add_argument("--limit", type=int, default=20, help="Limit the number of results returned. If 0 only statistics are returned")
SEARCH_PARSER.add_argument("--exact", action='store_true', help="Validate search results")

HUNT_PARSER = ARG_SUBPARSERS.add_parser('hunt', help='Hunt for files using YARA rules')
HUNT_PARSER.add_argument('yarafile', type=str)

SIGN_PARSER = ARG_SUBPARSERS.add_parser('sign', help="Generate IOC on samples")
SIGN_PARSER.add_argument("files", type=str, nargs='+', help="Files/Hashes (md5/sha1/sha256) to send to signer")

SIGN_PARSER.add_argument("--patternCount", "-c", type=int, default=3, help="Specify the number of fragments in a generated rule", dest='fragcount')
SIGN_PARSER.add_argument("--strategy", "-s", type=str, choices=['none', 'strict'], help="Specify if the signature should be extracted from full file (none) or a subset (strict)", default='none')
SIGN_PARSER.add_argument("--cluster", help="Treat files as a cluster in order to minimize the number of generated signatures", action='store_true')
SIGN_PARSER.add_argument("--other", nargs='*', help="Specify additional options to send, in the form of a tuple (key, value)", default=[], action='store')

SIGN_PARSER.add_argument("--u", type=bool, help='Upload file(s) if missing', default=True)
SIGN_PARSER.add_argument("--yara", help='Dump generated YARA signatures to screen', default=False, action="store_true")

CLASSIFY_PARSER = ARG_SUBPARSERS.add_parser('classify', help="Classify samples using Machine Learning")
CLASSIFY_PARSER.add_argument("files", type=str, nargs='+')
CLASSIFY_PARSER.add_argument("-u", type=bool, help='Upload file(s) if missing', default=True)

FILEINFO_PARSER = ARG_SUBPARSERS.add_parser('metadata', help="Retrieve file metadata")
FILEINFO_PARSER.add_argument("filehash", type=str, help="File hash (md5/sha1/sha256) to retrieve metadata")

USAGE_PARSER = ARG_SUBPARSERS.add_parser('demo', help="Show usage examples")

LABEL_COLOR = {
    'clean':Style.BRIGHT + Fore.GREEN,
    'malware':Style.BRIGHT + Fore.RED,
    'pua':Style.BRIGHT + Fore.YELLOW,
    'unknown':Style.BRIGHT + Fore.CYAN,
    'suspicious':Style.BRIGHT + Fore.MAGENTA
}

def dump(obj, nested_level=0, output=sys.stdout):
    spacing = '   '
    if isinstance(obj, dict):
        print >> output, '%s{' % ((nested_level) * spacing)
        for key, value in obj.items():
            if hasattr(value, '__iter__'):
                print >> output, '%s%s:' % ((nested_level + 1) * spacing, key)
                dump(value, nested_level + 1, output)
            else:
                print >> output, '%s%s: %s' % ((nested_level + 1) * spacing, key, value)
        print >> output, '%s}' % (nested_level * spacing)
    elif isinstance(obj, list):
        print >> output, '%s[' % ((nested_level) * spacing)
        for value in obj:
            if hasattr(value, '__iter__'):
                dump(value, nested_level + 1, output)
            else:
                print >> output, '%s%s' % ((nested_level + 1) * spacing, value)
        print >> output, '%s]' % ((nested_level) * spacing)
    else:
        print >> output, '%s%s' % (nested_level * spacing, obj)

def smart_size(size):
    if not isinstance(size, int):
        return size

    if size >= 1024*1024*1024:
        return "{0:.2f} GB".format(float(size)/(1024*1024*1024))
    elif size >= 1024*1024:
        return "{0:.2f}MB".format(float(size)/(1024*1024))
    elif size > 1024:
        return "{0:.2f} KB".format(float(size)/1024)
    else:
        return "{0:B}".format(size)

def get_filelist(dirname):
    return [x for x in glob.glob(os.path.join(dirname, '*')) if os.path.isfile(x)]

def show_row(val):
    color = Fore.WHITE
    label = None
    if val.has_key(u'label'):
        color = LABEL_COLOR.get(val['label'], Fore.WHITE)
        label = val[u'label']

    size = smart_size(val.get(u'size', "N/A"))
    print("SHA1:{0}{1}{2} Label:{3}{4:11}{2} Family:{3}{5:10} {2}Size:{0}{6}{7}".format(\
        Style.BRIGHT, val['sha1'],
        Style.RESET_ALL, color, label.title(),
        val.get(u'family', "N/A").title(),
        Style.BRIGHT + Fore.WHITE, size))

def show_results(results):
    print("-"*100)
    for val in results:
        show_row(val)

def show_stats(stats):
    print("Found {0} results : {1}{2} clean {3}{4} malware {5}{6} PUA {7}{8} unknown {9}{10} suspicious".format(
        stats['total_count'],
        LABEL_COLOR['clean'], stats['clean_count'],
        LABEL_COLOR['malware'], stats['malware_count'],<commit_msg>Fix exception when displaying files with less than 1 KB<commit_after>import os
import sys
import json
import datetime
import argparse
import glob
try:
    from colorama import init, Fore, Back, Style
except ImportError:
    print("Error importing colorama. Please make sure you have it (pip install colorama)")
    sys.exit(-1)

try:
    from BinarlyAPIv1 import BinarlyAPI, hex_pattern, ascii_pattern, wide_pattern, build_query
except ImportError:
    print("Error importing BinarlyAPI. You can find it here https://github.com/binarlyhq/binarly-sdk")
    sys.exit(-1)

BINOBJ = None
APIKEYFILENAME = 'apikey.txt'
APIKEYPATH = os.path.join(os.path.dirname(__file__), APIKEYFILENAME)

APIKEY = ''

ARGPARSER = argparse.ArgumentParser(description='Binarly API Query')
ARGPARSER.add_argument("--key", "-k", help="Binarly APIKey", default='')
ARGPARSER.add_argument("--usehttp", "-u", help="Use HTTP instead of HTTPS when communicating. By default HTTPS is used.", action="store_true")
ARG_SUBPARSERS = ARGPARSER.add_subparsers(help='commands', dest='commands')

SEARCH_PARSER = ARG_SUBPARSERS.add_parser('search', help="Search arbitrary hex patterns")
SEARCH_PARSER.add_argument("hex", type=str, nargs='*', default=[])
SEARCH_PARSER.add_argument("-a", type=str, nargs='*', help="ASCII string to search", default=[])
SEARCH_PARSER.add_argument("-w", type=str, nargs='*', help="WIDE string to search", default=[])
SEARCH_PARSER.add_argument("--limit", type=int, default=20, help="Limit the number of results returned. If 0 only statistics are returned")
SEARCH_PARSER.add_argument("--exact", action='store_true', help="Validate search results")

HUNT_PARSER = ARG_SUBPARSERS.add_parser('hunt', help='Hunt for files using YARA rules')
HUNT_PARSER.add_argument('yarafile', type=str)

SIGN_PARSER = ARG_SUBPARSERS.add_parser('sign', help="Generate IOC on samples")
SIGN_PARSER.add_argument("files", type=str, nargs='+', help="Files/Hashes (md5/sha1/sha256) to send to signer")

SIGN_PARSER.add_argument("--patternCount", "-c", type=int, default=3, help="Specify the number of fragments in a generated rule", dest='fragcount')
SIGN_PARSER.add_argument("--strategy", "-s", type=str, choices=['none', 'strict'], help="Specify if the signature should be extracted from full file (none) or a subset (strict)", default='none')
SIGN_PARSER.add_argument("--cluster", help="Treat files as a cluster in order to minimize the number of generated signatures", action='store_true')
SIGN_PARSER.add_argument("--other", nargs='*', help="Specify additional options to send, in the form of a tuple (key, value)", default=[], action='store')

SIGN_PARSER.add_argument("--u", type=bool, help='Upload file(s) if missing', default=True)
SIGN_PARSER.add_argument("--yara", help='Dump generated YARA signatures to screen', default=False, action="store_true")

CLASSIFY_PARSER = ARG_SUBPARSERS.add_parser('classify', help="Classify samples using Machine Learning")
CLASSIFY_PARSER.add_argument("files", type=str, nargs='+')
CLASSIFY_PARSER.add_argument("-u", type=bool, help='Upload file(s) if missing', default=True)

FILEINFO_PARSER = ARG_SUBPARSERS.add_parser('metadata', help="Retrieve file metadata")
FILEINFO_PARSER.add_argument("filehash", type=str, help="File hash (md5/sha1/sha256) to retrieve metadata")

USAGE_PARSER = ARG_SUBPARSERS.add_parser('demo', help="Show usage examples")

LABEL_COLOR = {
    'clean':Style.BRIGHT + Fore.GREEN,
    'malware':Style.BRIGHT + Fore.RED,
    'pua':Style.BRIGHT + Fore.YELLOW,
    'unknown':Style.BRIGHT + Fore.CYAN,
    'suspicious':Style.BRIGHT + Fore.MAGENTA
}

def dump(obj, nested_level=0, output=sys.stdout):
    spacing = '   '
    if isinstance(obj, dict):
        print >> output, '%s{' % ((nested_level) * spacing)
        for key, value in obj.items():
            if hasattr(value, '__iter__'):
                print >> output, '%s%s:' % ((nested_level + 1) * spacing, key)
                dump(value, nested_level + 1, output)
            else:
                print >> output, '%s%s: %s' % ((nested_level + 1) * spacing, key, value)
        print >> output, '%s}' % (nested_level * spacing)
    elif isinstance(obj, list):
        print >> output, '%s[' % ((nested_level) * spacing)
        for value in obj:
            if hasattr(value, '__iter__'):
                dump(value, nested_level + 1, output)
            else:
                print >> output, '%s%s' % ((nested_level + 1) * spacing, value)
        print >> output, '%s]' % ((nested_level) * spacing)
    else:
        print >> output, '%s%s' % (nested_level * spacing, obj)

def smart_size(size):
    if not isinstance(size, int):
        return size

    if size >= 1024*1024*1024:
        return "{0:.2f} GB".format(float(size)/(1024*1024*1024))
    elif size >= 1024*1024:
        return "{0:.2f}MB".format(float(size)/(1024*1024))
    elif size > 1024:
        return "{0:.2f} KB".format(float(size)/1024)
    else:
        return "{0} B".format(size)

def get_filelist(dirname):
    return [x for x in glob.glob(os.path.join(dirname, '*')) if os.path.isfile(x)]

def show_row(val):
    color = Fore.WHITE
    label = None
    if val.has_key(u'label'):
        color = LABEL_COLOR.get(val['label'], Fore.WHITE)
        label = val[u'label']

    size = smart_size(val.get(u'size', "N/A"))
    print("SHA1:{0}{1}{2} Label:{3}{4:11}{2} Family:{3}{5:10} {2}Size:{0}{6}{7}".format(\
        Style.BRIGHT, val['sha1'],
        Style.RESET_ALL, color, label.title(),
        val.get(u'family', "N/A").title(),
        Style.BRIGHT + Fore.WHITE, size))

def show_results(results):
    print("-"*100)
    for val in results:
        show_row(val)

def show_stats(stats):
    print("Found {0} results : {1}{2} clean {3}{4} malware {5}{6} PUA {7}{8} unknown {9}{10} suspicious".format(
        stats['total_count'],
        LABEL_COLOR['clean'], stats['clean_count'],
        LABEL_COLOR['malware'], stats['malware_count'],
{'fraction_non_alphanumeric': 0.15865874363327673, 'fraction_numerical': 0.018930390492359932, 'mean_word_length': 4.313937753721245, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7009831', 'n_tokens_mistral': 4044, 'n_tokens_neox': 3924, 'n_words': 957}