<commit_before>#!/usr/bin/env python

import logging
import datetime
from collections import namedtuple

import pytz
import json
import iso8601
from lxml import etree
from lxml.etree import XMLSyntaxError

from Evtx.Evtx import Evtx
from Evtx.Views import evtx_file_xml_view


g_logger = logging.getLogger("process-forest.global")


def to_lxml(record_xml):
    """
    @type record: Record
    """
    return etree.fromstring("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>%s" %
            record_xml.replace("xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"", ""))


class Process(object):
    NOTE_FAKE_PARENT = "Fake Parent: This is a faked process created since a ppid didn't exist"
    NOTE_END_LOST = "Lost End Timestamp: This end timestamp is suspect, because it collided with another process"
    def __init__(self, pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer):
        super(Process, self).__init__()
        self.pid = pid
        self.ppid = ppid
        self.path = path
        self.cmdline = cmdline
        self.ppname = ppname
        self.hashes = hashes
        self.user = user
        self.domain = domain
        self.logonid = logonid
        self.computer = computer
        self.begin = datetime.datetime.min
        self.end = datetime.datetime.min
        self.parent = None
        self.children = []
        self.notes = None
        self.id = None  # set by analyzer, unique with analyzer session

    def __str__(self):
        return "Process(%s, cmd=%s, hashes=%s, pid=%x, ppid=%x, begin=%s, end=%s" % (
                self.path, self.cmdline, self.hashes, self.pid, self.ppid,
                self.begin.isoformat(), self.end.isoformat())

    # TODO: move serialize, deserialize here


def create_fake_parent_process(pid, name):
    p = Process(pid, 0, "UNKNOWN", "UNKNOWN", "UNKNOWN", name, "UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN")
    p.notes = Process.NOTE_FAKE_PARENT
    return p


class NotAProcessEventError(Exception):
    pass


class Entry(object):
    def __init__(self, xml, record):
        super(Entry, self).__init__()
        self._xml = xml
        self._record = record
        self._node = to_lxml(self._xml)
        self._logger = logging.getLogger("process-forest.Entry")

    def get_xpath(self, path):
        return self._node.xpath(path)[0]

    def get_eid(self):
        return int(self.get_xpath("/Event/System/EventID").text)

    def get_timestamp(self):
        return self._record.timestamp()

    def is_process_created_event(self):
        return self.get_eid() == 4688

    def is_process_exited_event(self):
        return self.get_eid() == 4689

    def is_sysmon_proc_created_event(self):
        return self.get_eid() == 1

    def is_sysmon_proc_exited_event(self):
        return self.get_eid() == 5

    def get_process_from_4688_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='NewProcessName']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='NewProcessId']").text, 0x10)
        ppid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        try:
            cmdline = self.get_xpath("/Event/EventData/Data[@Name='CommandLine']").text
        except:
            cmdline = "UNKNOWN"
        try:
            ppname = self.get_xpath("/Event/EventData/Data[@Name='ParentProcessName']").text
        except:
            ppname = "UNKNOWN"
        hashes = "UNKNOWN"
        user = self.get_xpath("/Event/EventData/Data[@Name='SubjectUserName']").text
        domain = self.get_xpath("/Event/EventData/Data[@Name='SubjectDomainName']").text
        logonid = self.get_xpath("/Event/EventData/Data[@Name='SubjectLogonId']").text
        computer = self.get_xpath("/Event/System/Computer").text
        p = Process(pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer)
        p.begin = self._record.timestamp()
        return p

    def get_process_from_4689_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='ProcessName']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        ppid = int(self.get_xpath("/Event/System/Execution").get("ProcessID"), 10)
        cmdline = "UNKNOWN"
        ppname = "UNKNOWN"
        hashes = "UNKNOWN"
        user = self.get_xpath("/Event/EventData/Data[@Name='SubjectUserName']").text
        domain = self.get_xpath("/Event/EventData/Data[@Name='SubjectDomainName']").text
        logonid = self.get_xpath("/Event/EventData/Data[@Name='SubjectLogonId']").text
        computer = self.get_xpath("/Event/System/Computer").text
        p = Process(pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer)
        p.end = self._record.timestamp()
        return p
        
    def get_process_from_1_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='Image']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        ppid = int(self.get_xpath("/Event/EventData/Data[@Name='ParentProcessId']").text, 0x10)
        cmdline = self.get_xpath("/Event/EventData/Data[@Name='CommandLine']").text
        ppname = self.get_xpath("/Event/EventData/Data[@Name='ParentImage']").text
        hashes = self.get_xpath("/Event/EventData/Data[@Name='Hashes']").text
        ud = self.get_xpath("/Event/EventData/Data[@Name='User']").text.split("\\")
        user = ud[1]
        domain = ud[0]
        logonid = self.get_xpath("/Event/EventData/Data[@Name='LogonId']").text
        computer = self.get_xpath("/Event/System/Computer").text
        p = Process(pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer)
        p.begin = self._record.timestamp()
        return p
    
    def get_process_from_5_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='Image']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        ppid = 0
        cmdline = "UNKNOWN"
        ppname = "UNKNOWN"
        hashes = "UNKNOWN"
        user = "UNKNOWN"
        domain = "UNKNOWN"
        logonid = "UNKNOWN"
        computer = self.get_xpath("/Event/System/Computer").text<commit_msg>set lxml parser to utf-8<commit_after>#!/usr/bin/env python

import logging
import datetime
from collections import namedtuple

import pytz
import json
import iso8601
from lxml import etree
from lxml.etree import XMLSyntaxError

from Evtx.Evtx import Evtx
from Evtx.Views import evtx_file_xml_view


g_logger = logging.getLogger("process-forest.global")


def to_lxml(record_xml):
    """
    @type record: Record
    """
    utf8_parser = etree.XMLParser(encoding='utf-8')
    return etree.fromstring("<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\" ?>%s" %
            record_xml.replace("xmlns=\"http://schemas.microsoft.com/win/2004/08/events/event\"", "").encode('utf-8'), parser=utf8_parser)


class Process(object):
    NOTE_FAKE_PARENT = "Fake Parent: This is a faked process created since a ppid didn't exist"
    NOTE_END_LOST = "Lost End Timestamp: This end timestamp is suspect, because it collided with another process"
    def __init__(self, pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer):
        super(Process, self).__init__()
        self.pid = pid
        self.ppid = ppid
        self.path = path
        self.cmdline = cmdline
        self.ppname = ppname
        self.hashes = hashes
        self.user = user
        self.domain = domain
        self.logonid = logonid
        self.computer = computer
        self.begin = datetime.datetime.min
        self.end = datetime.datetime.min
        self.parent = None
        self.children = []
        self.notes = None
        self.id = None  # set by analyzer, unique with analyzer session

    def __str__(self):
        return "Process(%s, cmd=%s, hashes=%s, pid=%x, ppid=%x, begin=%s, end=%s" % (
                self.path, self.cmdline, self.hashes, self.pid, self.ppid,
                self.begin.isoformat(), self.end.isoformat())

    # TODO: move serialize, deserialize here


def create_fake_parent_process(pid, name):
    p = Process(pid, 0, "UNKNOWN", "UNKNOWN", "UNKNOWN", name, "UNKNOWN", "UNKNOWN", "UNKNOWN", "UNKNOWN")
    p.notes = Process.NOTE_FAKE_PARENT
    return p


class NotAProcessEventError(Exception):
    pass


class Entry(object):
    def __init__(self, xml, record):
        super(Entry, self).__init__()
        self._xml = xml
        self._record = record
        self._node = to_lxml(self._xml)
        self._logger = logging.getLogger("process-forest.Entry")

    def get_xpath(self, path):
        return self._node.xpath(path)[0]

    def get_eid(self):
        return int(self.get_xpath("/Event/System/EventID").text)

    def get_timestamp(self):
        return self._record.timestamp()

    def is_process_created_event(self):
        return self.get_eid() == 4688

    def is_process_exited_event(self):
        return self.get_eid() == 4689

    def is_sysmon_proc_created_event(self):
        return self.get_eid() == 1

    def is_sysmon_proc_exited_event(self):
        return self.get_eid() == 5

    def get_process_from_4688_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='NewProcessName']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='NewProcessId']").text, 0x10)
        ppid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        try:
            cmdline = self.get_xpath("/Event/EventData/Data[@Name='CommandLine']").text
        except:
            cmdline = "UNKNOWN"
        try:
            ppname = self.get_xpath("/Event/EventData/Data[@Name='ParentProcessName']").text
        except:
            ppname = "UNKNOWN"
        hashes = "UNKNOWN"
        user = self.get_xpath("/Event/EventData/Data[@Name='SubjectUserName']").text
        domain = self.get_xpath("/Event/EventData/Data[@Name='SubjectDomainName']").text
        logonid = self.get_xpath("/Event/EventData/Data[@Name='SubjectLogonId']").text
        computer = self.get_xpath("/Event/System/Computer").text
        p = Process(pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer)
        p.begin = self._record.timestamp()
        return p

    def get_process_from_4689_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='ProcessName']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        ppid = int(self.get_xpath("/Event/System/Execution").get("ProcessID"), 10)
        cmdline = "UNKNOWN"
        ppname = "UNKNOWN"
        hashes = "UNKNOWN"
        user = self.get_xpath("/Event/EventData/Data[@Name='SubjectUserName']").text
        domain = self.get_xpath("/Event/EventData/Data[@Name='SubjectDomainName']").text
        logonid = self.get_xpath("/Event/EventData/Data[@Name='SubjectLogonId']").text
        computer = self.get_xpath("/Event/System/Computer").text
        p = Process(pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer)
        p.end = self._record.timestamp()
        return p

    def get_process_from_1_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='Image']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        ppid = int(self.get_xpath("/Event/EventData/Data[@Name='ParentProcessId']").text, 0x10)
        cmdline = self.get_xpath("/Event/EventData/Data[@Name='CommandLine']").text
        ppname = self.get_xpath("/Event/EventData/Data[@Name='ParentImage']").text
        hashes = self.get_xpath("/Event/EventData/Data[@Name='Hashes']").text
        ud = self.get_xpath("/Event/EventData/Data[@Name='User']").text.split("\\")
        user = ud[1]
        domain = ud[0]
        logonid = self.get_xpath("/Event/EventData/Data[@Name='LogonId']").text
        computer = self.get_xpath("/Event/System/Computer").text
        p = Process(pid, ppid, cmdline, ppname, hashes, path, user, domain, logonid, computer)
        p.begin = self._record.timestamp()
        return p

    def get_process_from_5_event(self):
        path = self.get_xpath("/Event/EventData/Data[@Name='Image']").text
        pid = int(self.get_xpath("/Event/EventData/Data[@Name='ProcessId']").text, 0x10)
        ppid = 0
        cmdline = "UNKNOWN"
        ppname = "UNKNOWN"
        hashes = "UNKNOWN"
        user = "UNKNOWN"
        domain = "UNKNOWN"
        logonid = "UNKNOWN"
        computer = self.get_xpath("/Event/System/Computer").text
{'fraction_non_alphanumeric': 0.14351374910848722, 'fraction_numerical': 0.009588715429114826, 'mean_word_length': 2.9810725552050474, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 2, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 27}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7019894', 'n_tokens_mistral': 4128, 'n_tokens_neox': 4019, 'n_words': 749}