# Path: apiscout/FingerprintCrawler.py
# class FingerprintCrawler():
# 
#     def __init__(self, db_paths, winapi1024_path, root_path, filter_size=0):
#         self._scout = self._initScout(db_paths, winapi1024_path)
#         self._filter_size = filter_size
#         self._root_path = root_path
#         self._ignore_files = {}
#         self._empty_fingerprint = self.getEmptyFingerprint()
#         
#     def _initScout(self, db_paths, winapi1024_path):
#         scout = ApiScout()
#         # override potential ASLR offsets that are stored in the API DB files.
#         scout.ignoreAslrOffsets(True)
#         # load DB file
#         for db_path in db_paths:
#             scout.loadDbFile(db_path)
#         # load WinApi1024 vector
#         scout.loadWinApi1024(winapi1024_path)
#         return scout
#         
#     def _getFamilyAndPath(self, path_to_binary):
#         sub_path = path_to_binary[len(self._root_path):]
#         path_parts = [part for part in sub_path.split(os.sep) if part]
#         family = "-"
#         path = ""
#         if len(path_parts) > 1:
#             family = path_parts[0]
#             path = os.sep.join(path_parts[1:])
#         else:
#             path = path_parts[0]
#         return family, path
#         
#     def getEmptyFingerprint(self):
#         results = self._scout.crawl(b"\x00" * 0x100)
#         filtered_results = self._scout.filter(results, 0, 0, self._filter_size)
#         api_vectors = self._scout.getWinApi1024Vectors(filtered_results)
#         apiscout_db_name, primary_vector = self._scout.getPrimaryVector(api_vectors)
#         return primary_vector["vector"]
#         
#     def getFingerprint(self, path_to_binary):
#         family, path = self._getFamilyAndPath(path_to_binary)
#         if self._isIgnorableBinary(family, path):
#             return self._ignore_files[family + "_" + path]
#         if os.path.isfile(path_to_binary):
#             with open(path_to_binary, "rb") as f_binary:
#                 binary = f_binary.read()
#         # scout the binary
#         try:
#             results = self._scout.crawl(binary)
#             filtered_results = self._scout.filter(results, 0, 0, self._filter_size)
#             api_vectors = self._scout.getWinApi1024Vectors(filtered_results)
#             apiscout_db_name, primary_vector = self._scout.getPrimaryVector(api_vectors)
#             return (family, path, primary_vector["in_api_vector"], primary_vector["num_unique_apis"], primary_vector["vector"])
#         except:
#             return (family, path, 0, 0, self._empty_fingerprint)
#         
#     def loadIgnoreList(self, path_to_list):
#         self._ignore_files = {}
#         if os.path.isfile(path_to_list):
#             with open(path_to_list, "r") as infile:
#                 for line in infile.readlines():
#                     line = line.strip()
#                     if line:
#                         family = line.split(";")[0]
#                         binary_path = line.split(";")[1]
#                         self._ignore_files[family + "_" + binary_path] = tuple(line.split(";"))
#                         
#     def getNumberOfCrawlableFiles(self, file_list):
#         return len(file_list) - self.getNumberOfIgnorableFiles(file_list)
#         
#     def getNumberOfIgnorableFiles(self, file_list):
#         num_ignorable = 0
#         for path_to_binary in file_list:
#             family, path = self._getFamilyAndPath(path_to_binary)
#             if self._isIgnorableBinary(family, path):
#                 num_ignorable += 1
#         return num_ignorable
#     
#     def _isIgnorableBinary(self, family, path):
#         return family + "_" + path in self._ignore_files
#         
#     def persistFingerprintsToFile(self, fingerprint_list, dest_path):
#         with open(dest_path, "w") as f_destination:
#             for entry in fingerprint_list:
#                 f_destination.write("%s;%s;%s;%s;%s\n" % entry)


import argparse
import datetime
import os
import sys
import logging
from multiprocessing import Pool, cpu_count
from apiscout.FingerprintCrawler import FingerprintCrawler
    db_dir = get_this_dir() + os.sep + "dbs" + os.sep
    return [db_dir + fn for fn in os.listdir(db_dir) if fn.endswith(".json")]


def get_winapi1024_path():
    return get_this_dir() + os.sep + "apiscout" + os.sep + "data" + os.sep + "winapi1024v1.txt"
    

def get_file_list(root_path, required_part):
    file_list = []
    for root, subdir, files in os.walk(root_path):
        for filename in files:
            if required_part and required_part not in filename:
                continue
            file_list.append(os.path.join(root, filename))
    return file_list
    

def main():
    parser = argparse.ArgumentParser(description='Demo: Create a WinApi1024 fingerprint database.')
    parser.add_argument('root_path', type=str, default='', help='Directory to crawl recursively for building the database (malpedia folder structure assumed).')
    parser.add_argument('-f', '--filter_size', type=int, default=32, help='Filter out APIs that do not have a neighbour within N bytes (default: 32).')
    parser.add_argument('-r', '--path_require', type=str, default='', help='A string that files should contain in order to be considered for crawling (e.g. "_dump" for malpedia files).')
    parser.add_argument('-o', '--output_path', type=str, default='', help='Optionally save the fingerprint file to this location (instead of "./dbs/<YYYY-MM-DD>-winapi1024.csv".')
    parser.add_argument('-u', '--update_path', type=str, default='', help='Instead of generating a fingerprint database from scratch, load and update an existing file (add new entries only).')
    parser.add_argument('-i', '--ignore_empty', action='store_true', help='Do not store information for empty fingerprints.')

    args = parser.parse_args()
    if args.root_path and os.path.isdir(args.root_path):
        fingerprints = []

# Path: apiscout/FingerprintCrawler.py
# class FingerprintCrawler():
# 
#     def __init__(self, db_paths, winapi1024_path, root_path, filter_size=0):
#         self._scout = self._initScout(db_paths, winapi1024_path)
#         self._filter_size = filter_size
#         self._root_path = root_path
#         self._ignore_files = {}
#         self._empty_fingerprint = self.getEmptyFingerprint()
#         
#     def _initScout(self, db_paths, winapi1024_path):
#         scout = ApiScout()
#         # override potential ASLR offsets that are stored in the API DB files.
#         scout.ignoreAslrOffsets(True)
#         # load DB file
#         for db_path in db_paths:
#             scout.loadDbFile(db_path)
#         # load WinApi1024 vector
#         scout.loadWinApi1024(winapi1024_path)
#         return scout
#         
#     def _getFamilyAndPath(self, path_to_binary):
#         sub_path = path_to_binary[len(self._root_path):]
#         path_parts = [part for part in sub_path.split(os.sep) if part]
#         family = "-"
#         path = ""
#         if len(path_parts) > 1:
#             family = path_parts[0]
#             path = os.sep.join(path_parts[1:])
#         else:
#             path = path_parts[0]
#         return family, path
#         
#     def getEmptyFingerprint(self):
#         results = self._scout.crawl(b"\x00" * 0x100)
#         filtered_results = self._scout.filter(results, 0, 0, self._filter_size)
#         api_vectors = self._scout.getWinApi1024Vectors(filtered_results)
#         apiscout_db_name, primary_vector = self._scout.getPrimaryVector(api_vectors)
#         return primary_vector["vector"]
#         
#     def getFingerprint(self, path_to_binary):
#         family, path = self._getFamilyAndPath(path_to_binary)
#         if self._isIgnorableBinary(family, path):
#             return self._ignore_files[family + "_" + path]
#         if os.path.isfile(path_to_binary):
#             with open(path_to_binary, "rb") as f_binary:
#                 binary = f_binary.read()
#         # scout the binary
#         try:
#             results = self._scout.crawl(binary)
#             filtered_results = self._scout.filter(results, 0, 0, self._filter_size)
#             api_vectors = self._scout.getWinApi1024Vectors(filtered_results)
#             apiscout_db_name, primary_vector = self._scout.getPrimaryVector(api_vectors)
#             return (family, path, primary_vector["in_api_vector"], primary_vector["num_unique_apis"], primary_vector["vector"])
#         except:
#             return (family, path, 0, 0, self._empty_fingerprint)
#         
#     def loadIgnoreList(self, path_to_list):
#         self._ignore_files = {}
#         if os.path.isfile(path_to_list):
#             with open(path_to_list, "r") as infile:
#                 for line in infile.readlines():
#                     line = line.strip()
#                     if line:
#                         family = line.split(";")[0]
#                         binary_path = line.split(";")[1]
#                         self._ignore_files[family + "_" + binary_path] = tuple(line.split(";"))
#                         
#     def getNumberOfCrawlableFiles(self, file_list):
#         return len(file_list) - self.getNumberOfIgnorableFiles(file_list)
#         
#     def getNumberOfIgnorableFiles(self, file_list):
#         num_ignorable = 0
#         for path_to_binary in file_list:
#             family, path = self._getFamilyAndPath(path_to_binary)
#             if self._isIgnorableBinary(family, path):
#                 num_ignorable += 1
#         return num_ignorable
#     
#     def _isIgnorableBinary(self, family, path):
#         return family + "_" + path in self._ignore_files
#         
#     def persistFingerprintsToFile(self, fingerprint_list, dest_path):
#         with open(dest_path, "w") as f_destination:
#             for entry in fingerprint_list:
#                 f_destination.write("%s;%s;%s;%s;%s\n" % entry)


# Path: collect.py
import argparse
import datetime
import os
import sys
import logging
from multiprocessing import Pool, cpu_count
from apiscout.FingerprintCrawler import FingerprintCrawler

    db_dir = get_this_dir() + os.sep + "dbs" + os.sep
    return [db_dir + fn for fn in os.listdir(db_dir) if fn.endswith(".json")]


def get_winapi1024_path():
    return get_this_dir() + os.sep + "apiscout" + os.sep + "data" + os.sep + "winapi1024v1.txt"
    

def get_file_list(root_path, required_part):
    file_list = []
    for root, subdir, files in os.walk(root_path):
        for filename in files:
            if required_part and required_part not in filename:
                continue
            file_list.append(os.path.join(root, filename))
    return file_list
    

def main():
    parser = argparse.ArgumentParser(description='Demo: Create a WinApi1024 fingerprint database.')
    parser.add_argument('root_path', type=str, default='', help='Directory to crawl recursively for building the database (malpedia folder structure assumed).')
    parser.add_argument('-f', '--filter_size', type=int, default=32, help='Filter out APIs that do not have a neighbour within N bytes (default: 32).')
    parser.add_argument('-r', '--path_require', type=str, default='', help='A string that files should contain in order to be considered for crawling (e.g. "_dump" for malpedia files).')
    parser.add_argument('-o', '--output_path', type=str, default='', help='Optionally save the fingerprint file to this location (instead of "./dbs/<YYYY-MM-DD>-winapi1024.csv".')
    parser.add_argument('-u', '--update_path', type=str, default='', help='Instead of generating a fingerprint database from scratch, load and update an existing file (add new entries only).')
    parser.add_argument('-i', '--ignore_empty', action='store_true', help='Do not store information for empty fingerprints.')

    args = parser.parse_args()
    if args.root_path and os.path.isdir(args.root_path):
        fingerprints = []

        crawler = FingerprintCrawler(get_all_db_files(), get_winapi1024_path(), args.root_path, args.filter_size)