<commit_before>__author__ = 'matiasbevilacqua'

from settings import logger_getDebugMode
import logging
from mpEngineProdCons import MPEngineProdCons
from mpEngineWorker import MPEngineWorker
import Queue
import os
import re
import sqlite3
import ntpath
from contextlib import closing
import time
import struct
import hashlib
from appAux import update_progress, chunks, loadFile, psutil_phymem_usage, file_size
import appDB
import settings
from ShimCacheParser import read_mir, write_it
from AmCacheParser import _processAmCacheFile_StringIO
import zipfile
import contextlib
import xml.etree.ElementTree as ET
from datetime import timedelta, datetime
import sys
import traceback
import signal
import gc
import cProfile
from Ingest import appcompat_parsed
from Ingest import appcompat_mirregistryaudit
from Ingest import amcache_miracquisition
from Ingest import appcompat_mirlua_v1
from Ingest import appcompat_mirlua_v2
from Ingest import amcache_mirlua_v1
from Ingest import appcompat_csv
from Ingest import appcompat_redline
from Ingest import appcompat_raw_hive
from Ingest import appcompat_miracquisition
from Ingest import amcache_raw_hive
try:
    import pyregf
except ImportError:
    if settings.__PYREGF__:
        settings.__PYREGF__ = False
        print "Ooops seems you don't have pyregf!"
        print "AmCache loading support will be disabled"
else: settings.__PYREGF__ = True


logger = logging.getLogger(__name__)
_tasksPerJob = 10
supported_ingest_plugins = ['appcompat_parsed.Appcompat_parsed', 'amcache_miracquisition.Amcache_miracquisition',
                            'appcompat_mirregistryaudit.Appcompat_mirregistryaudit', 'amcache_mirlua_v1.Amcache_mirlua_v1',
                            'appcompat_mirlua_v2.Appcompat_mirlua_v2', 'appcompat_csv.Appcompat_csv',
                            'appcompat_redline.Appcompat_redline', 'appcompat_raw_hive.Appcompat_Raw_hive',
                            'appcompat_miracquisition.Appcompat_miracquisition', 'amcache_raw_hive.Amcache_Raw_hive']

# Load IngestTypes
ingest_plugins = {}
ingest_plugins_types_stack = []
for plugin in supported_ingest_plugins:
    ingest_plugins[eval(plugin).ingest_type] = eval(plugin)()
    ingest_plugins_types_stack.append(eval(plugin).ingest_type)


def do_cprofile(func):
    def profiled_func(*args, **kwargs):
        profile = cProfile.Profile()
        try:
            profile.enable()
            result = func(*args, **kwargs)
            profile.disable()
            return result
        finally:
            profile.print_stats()
    return profiled_func


class appLoadProd(MPEngineWorker):

    def do_work(self, next_task):
        self.logger.debug("do_work")
        rowsData = next_task()

        # Sanityzing entries
        for x in rowsData:
            # Check if we've been killed
            self.check_killed()
            sanityCheckOK = True
            # todo: Bring back here sanity check on AMCache dates (as the SQLite driver would die later when querying invalid dates)
            try:
                if sanityCheckOK:
                    # todo: Maybe we don't need this after the ISO patch to ShimCacheParser?
                    if x.LastModified != "N/A" and x.LastModified != None:
                        x.LastModified = datetime.strptime(x.LastModified, "%Y-%m-%d %H:%M:%S")
                    else:
                        x.LastModified = datetime.min

                    if x.LastUpdate != "N/A" and x.LastUpdate != None:
                        x.LastUpdate = datetime.strptime(x.LastUpdate, "%Y-%m-%d %H:%M:%S")
                    else:
                        x.LastUpdate = datetime.min

                    # We use FirstRun as LastModified for AmCache entries
                    if x.EntryType == settings.__AMCACHE__:
                        x.LastModified = x.FirstRun

                    # We use Modified2 as LastUpdate for AmCache entries
                    if x.EntryType == settings.__AMCACHE__:
                        x.LastUpdate = x.Modified2

                    # Sanitize things up (AmCache is full of these 'empty' entries which I don't have a clue what they are yet)
                    if x.FilePath is None:
                        x.FilePath = "None"
                    else:
                        x.FilePath = x.FilePath.replace("'", "''")
                        # Trim out UNC path prefix
                        x.FilePath = x.FilePath.replace("\\??\\", "")
                        # Trim out SYSVOL path prefix
                        x.FilePath = x.FilePath.replace("SYSVOL", "C:")
                    if x.FileName is None:
                        x.FileName = "None"
                    else:
                        x.FileName = x.FileName.replace("'", "''")
                else:
                    rowsData.remove(x)
            except Exception as e:
                self.logger.warning("Exception processing row (%s): %s" % (e.message, x))
                sanityCheckOK = False
                pass
        return rowsData


class appLoadCons(MPEngineWorker):

    def run(self):
        # Note: __init__ runs on multiprocessing's main thread and as such we can't use that to init a sqlite connection
        assert(len(self.extra_arg_list) == 1)
        self.dbfilenameFullPath = self.extra_arg_list[0]
        self.DB = None
        self.conn = None
<commit_msg>Improved sanity checking to AmCache timestamps.<commit_after>__author__ = 'matiasbevilacqua'

from settings import logger_getDebugMode
import logging
from mpEngineProdCons import MPEngineProdCons
from mpEngineWorker import MPEngineWorker
import Queue
import os
import re
import sqlite3
import ntpath
from contextlib import closing
import time
import struct
import hashlib
from appAux import update_progress, chunks, loadFile, psutil_phymem_usage, file_size
import appDB
import settings
from ShimCacheParser import read_mir, write_it
from AmCacheParser import _processAmCacheFile_StringIO
import zipfile
import contextlib
import xml.etree.ElementTree as ET
from datetime import timedelta, datetime
import sys
import traceback
import signal
import gc
import cProfile
from Ingest import appcompat_parsed
from Ingest import appcompat_mirregistryaudit
from Ingest import amcache_miracquisition
from Ingest import appcompat_mirlua_v1
from Ingest import appcompat_mirlua_v2
from Ingest import amcache_mirlua_v1
from Ingest import appcompat_csv
from Ingest import appcompat_redline
from Ingest import appcompat_raw_hive
from Ingest import appcompat_miracquisition
from Ingest import amcache_raw_hive
try:
    import pyregf
except ImportError:
    if settings.__PYREGF__:
        settings.__PYREGF__ = False
        print "Ooops seems you don't have pyregf!"
        print "AmCache loading support will be disabled"
else: settings.__PYREGF__ = True


logger = logging.getLogger(__name__)
_tasksPerJob = 10
supported_ingest_plugins = ['appcompat_parsed.Appcompat_parsed', 'amcache_miracquisition.Amcache_miracquisition',
                            'appcompat_mirregistryaudit.Appcompat_mirregistryaudit', 'amcache_mirlua_v1.Amcache_mirlua_v1',
                            'appcompat_mirlua_v2.Appcompat_mirlua_v2', 'appcompat_csv.Appcompat_csv',
                            'appcompat_redline.Appcompat_redline', 'appcompat_raw_hive.Appcompat_Raw_hive',
                            'appcompat_miracquisition.Appcompat_miracquisition', 'amcache_raw_hive.Amcache_Raw_hive']

# Load IngestTypes
ingest_plugins = {}
ingest_plugins_types_stack = []
for plugin in supported_ingest_plugins:
    ingest_plugins[eval(plugin).ingest_type] = eval(plugin)()
    ingest_plugins_types_stack.append(eval(plugin).ingest_type)


def do_cprofile(func):
    def profiled_func(*args, **kwargs):
        profile = cProfile.Profile()
        try:
            profile.enable()
            result = func(*args, **kwargs)
            profile.disable()
            return result
        finally:
            profile.print_stats()
    return profiled_func


class appLoadProd(MPEngineWorker):

    def _notInRange(self, start, end, x):
        """Return true if x is in the range [start, end]"""
        if start <= end:
            return not (start <= x <= end)
        else:
            return not (start <= x or x <= end)

    def do_work(self, next_task):
        self.logger.debug("do_work")
        rowsData = next_task()

        # Sanityzing entries
        for x in rowsData:
            # Check if we've been killed
            self.check_killed()
            sanityCheckOK = True
            # todo: Bring back here sanity check on AMCache dates (as the SQLite driver would die later when querying invalid dates)
            try:
                # We use FirstRun as LastModified for AmCache entries
                if x.EntryType == settings.__AMCACHE__:
                    x.LastModified = x.FirstRun

                # We use Modified2 as LastUpdate for AmCache entries
                if x.EntryType == settings.__AMCACHE__:
                    x.LastUpdate = x.Modified2

                if type(x.LastModified) != datetime:
                    # todo: Maybe we don't need this after the ISO patch to ShimCacheParser?
                    if x.LastModified != "N/A" and x.LastModified != None:
                        x.LastModified = datetime.strptime(x.LastModified, "%Y-%m-%d %H:%M:%S")
                    else:
                        x.LastModified = datetime.min

                if type(x.LastModified) != datetime:
                    if x.LastUpdate != "N/A" and x.LastUpdate != None:
                        x.LastUpdate = datetime.strptime(x.LastUpdate, "%Y-%m-%d %H:%M:%S")
                    else:
                        x.LastUpdate = datetime.min

                # Sanitize things up (AmCache is full of these 'empty' entries which I don't have a clue what they are yet)
                if x.FilePath is None:
                    x.FilePath = "None"
                else:
                    x.FilePath = x.FilePath.replace("'", "''")
                    # Trim out UNC path prefix
                    x.FilePath = x.FilePath.replace("\\??\\", "")
                    # Trim out SYSVOL path prefix
                    x.FilePath = x.FilePath.replace("SYSVOL", "C:")
                if x.FileName is None:
                    x.FileName = "None"
                else:
                    x.FileName = x.FileName.replace("'", "''")

                if x.EntryType == settings.__AMCACHE__:
                    # Sanity check AMCache dates:
                    # We need to exclude these entries as the SQLite driver would die later when queried
                    minSQLiteDTS = datetime(1, 1, 1, 0, 0, 0)
                    maxSQLiteDTS = datetime(9999, 12, 31, 0, 0, 0)

                    if self._notInRange(minSQLiteDTS, maxSQLiteDTS, x.FirstRun):
                        sanityCheckOK = False
                        settings.logger.warning(
                            "Weird FirstRun date, skipping entry: %s - %s - %s" % (x.HostID, x.FilePath, x.FirstRun))
                    if self._notInRange(minSQLiteDTS, maxSQLiteDTS, x.Modified1):
                        sanityCheckOK = False
                        settings.logger.warning(
                            "Weird Modified1 date, ignoring as this will kill sqlite on query: %s - %s - %s" % (
                            x.HostID, x.FilePath, x.Modified1))
                    if self._notInRange(minSQLiteDTS, maxSQLiteDTS, x.Modified2):
                        sanityCheckOK = False
                        settings.logger.warning(
                            "Weird Modified2 date, ignoring as this will kill sqlite on query: %s - %s - %s" % (
                            x.HostID, x.FilePath, x.Modified2))
                    if self._notInRange(minSQLiteDTS, maxSQLiteDTS, x.LinkerTS):
                        sanityCheckOK = False
                        settings.logger.warning(
                            "Weird LinkerTS date, ignoring as this will kill sqlite on query: %s - %s - %s" % (
                            x.HostID, x.FilePath, x.LinkerTS))

                if not sanityCheckOK:
                    rowsData.remove(x)
            except Exception as e:
                exc_type, exc_obj, exc_tb = sys.exc_info()
                fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
                self.logger.warning("Exception processing row (%s): %s [%s / %s / %s]" % (e.message, x, exc_type, fname, exc_tb.tb_lineno))
                pass
        return rowsData


class appLoadCons(MPEngineWorker):

    def run(self):
        # Note: __init__ runs on multiprocessing's main thread and as such we can't use that to init a sqlite connection
        assert(len(self.extra_arg_list) == 1)
        self.dbfilenameFullPath = self.extra_arg_list[0]
        self.DB = None
        self.conn = None

{'fraction_non_alphanumeric': 0.07492414222360538, 'fraction_numerical': 0.0040457480743795226, 'mean_word_length': 2.062663807481534, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3976957', 'n_tokens_mistral': 3684, 'n_tokens_neox': 3504, 'n_words': 990}