<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2014-2016 <NAME> (@stamparm)
See the file 'LICENSE' for copying permission
"""

import os
import signal
import socket
import SocketServer
import sys
import threading
import time
import traceback

from core.common import check_sudo
from core.settings import config
from core.settings import CONDENSE_ON_TRAIL_KEYWORDS
from core.settings import CONDENSED_EVENTS_FLUSH_PERIOD
from core.settings import DEFAULT_ERROR_LOG_PERMISSIONS
from core.settings import DEFAULT_EVENT_LOG_PERMISSIONS
from core.settings import TIME_FORMAT
from core.settings import WHITELIST

_thread_data = threading.local()

def create_log_directory():
    if not os.path.isdir(config.LOG_DIR):
        if check_sudo() is False:
            exit("[!] please run with sudo/Administrator privileges")
        os.makedirs(config.LOG_DIR)
    print("[i] using '%s' for log storage" % config.LOG_DIR)

def get_event_log_handle(sec, flags=os.O_APPEND | os.O_CREAT | os.O_WRONLY):
    localtime = time.localtime(sec)
    _ = os.path.join(config.LOG_DIR, "%d-%02d-%02d.log" % (localtime.tm_year, localtime.tm_mon, localtime.tm_mday))
    if _ != getattr(_thread_data, "event_log_path", None):
        if not os.path.exists(_):
            open(_, "w+").close()
            os.chmod(_, DEFAULT_EVENT_LOG_PERMISSIONS)
        _thread_data.event_log_path = _
        _thread_data.event_log_handle = os.open(_thread_data.event_log_path, flags)
    return _thread_data.event_log_handle

def get_error_log_handle(flags=os.O_APPEND | os.O_CREAT | os.O_WRONLY):
    if not hasattr(_thread_data, "error_log_handle"):
        _ = os.path.join(config.LOG_DIR, "error.log")
        if not os.path.exists(_):
            open(_, "w+").close()
            os.chmod(_, DEFAULT_ERROR_LOG_PERMISSIONS)
        _thread_data.error_log_path = _
        _thread_data.error_log_handle = os.open(_thread_data.error_log_path, flags)
    return _thread_data.error_log_handle

def safe_value(value):
    retval = str(value or '-')
    if any(_ in retval for _ in (' ', '"')):
        retval = "\"%s\"" % retval.replace('"', '""')
    return retval

def log_event(event_tuple, packet=None, skip_write=False, skip_condensing=False):
    try:
        sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info = event_tuple[:10]
        if not any(_ in WHITELIST for _ in (src_ip, dst_ip)):
            if not skip_write:
                localtime = "%s.%06d" % (time.strftime(TIME_FORMAT, time.localtime(int(sec))), usec)

                if not skip_condensing:
                    if (sec - getattr(_thread_data, "condensed_events_flush_sec", 0)) > CONDENSED_EVENTS_FLUSH_PERIOD:
                        _thread_data.condensed_events_flush_sec = sec

                        for key in getattr(_thread_data, "condensed_events", []):
                            condensed = False

                            _ = _thread_data.condensed_events[key][0]
                            condensed_event = list(_)

                            for i in xrange(1, len(_thread_data.condensed_events[key])):
                                _ = _thread_data.condensed_events[key][i]
                                for j in xrange(3, 7):  # src_port, dst_ip, dst_port, proto
                                    if _[j] != condensed_event[j]:
                                        condensed = True
                                        if not isinstance(condensed_event[j], set):
                                            condensed_event[j] = set((condensed_event[j],))
                                        condensed_event[j].add(_[j])

                            if condensed:
                                for i in xrange(len(condensed_event)):
                                    if isinstance(condensed_event[i], set):
                                        condensed_event[i] = ','.join(str(_) for _ in sorted(condensed_event[i]))

                            log_event(condensed_event, skip_condensing=True)

                        _thread_data.condensed_events = {}

                    if any(_ in info for _ in CONDENSE_ON_TRAIL_KEYWORDS):
                        if not hasattr(_thread_data, "condensed_events"):
                            _thread_data.condensed_events = {}
                        key = (src_ip, trail)
                        if key not in _thread_data.condensed_events:
                            _thread_data.condensed_events[key] = []
                        _thread_data.condensed_events[key].append(event_tuple)<commit_msg>Minor style update<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2014-2016 <NAME> (@stamparm)
See the file 'LICENSE' for copying permission
"""

import os
import signal
import socket
import SocketServer
import sys
import threading
import time
import traceback

from core.common import check_sudo
from core.settings import config
from core.settings import CONDENSE_ON_TRAIL_KEYWORDS
from core.settings import CONDENSED_EVENTS_FLUSH_PERIOD
from core.settings import DEFAULT_ERROR_LOG_PERMISSIONS
from core.settings import DEFAULT_EVENT_LOG_PERMISSIONS
from core.settings import TIME_FORMAT
from core.settings import WHITELIST

_thread_data = threading.local()

def create_log_directory():
    if not os.path.isdir(config.LOG_DIR):
        if check_sudo() is False:
            exit("[!] please run with sudo/Administrator privileges")
        os.makedirs(config.LOG_DIR)
    print("[i] using '%s' for log storage" % config.LOG_DIR)

def get_event_log_handle(sec, flags=os.O_APPEND | os.O_CREAT | os.O_WRONLY):
    localtime = time.localtime(sec)
    _ = os.path.join(config.LOG_DIR, "%d-%02d-%02d.log" % (localtime.tm_year, localtime.tm_mon, localtime.tm_mday))
    if _ != getattr(_thread_data, "event_log_path", None):
        if not os.path.exists(_):
            open(_, "w+").close()
            os.chmod(_, DEFAULT_EVENT_LOG_PERMISSIONS)
        _thread_data.event_log_path = _
        _thread_data.event_log_handle = os.open(_thread_data.event_log_path, flags)
    return _thread_data.event_log_handle

def get_error_log_handle(flags=os.O_APPEND | os.O_CREAT | os.O_WRONLY):
    if not hasattr(_thread_data, "error_log_handle"):
        _ = os.path.join(config.LOG_DIR, "error.log")
        if not os.path.exists(_):
            open(_, "w+").close()
            os.chmod(_, DEFAULT_ERROR_LOG_PERMISSIONS)
        _thread_data.error_log_path = _
        _thread_data.error_log_handle = os.open(_thread_data.error_log_path, flags)
    return _thread_data.error_log_handle

def safe_value(value):
    retval = str(value or '-')
    if any(_ in retval for _ in (' ', '"')):
        retval = "\"%s\"" % retval.replace('"', '""')
    return retval

def log_event(event_tuple, packet=None, skip_write=False, skip_condensing=False):
    try:
        sec, usec, src_ip, src_port, dst_ip, dst_port, proto, trail_type, trail, info = event_tuple[:10]
        if not any(_ in WHITELIST for _ in (src_ip, dst_ip)):
            if not skip_write:
                localtime = "%s.%06d" % (time.strftime(TIME_FORMAT, time.localtime(int(sec))), usec)

                if not skip_condensing:
                    if (sec - getattr(_thread_data, "condensed_events_flush_sec", 0)) > CONDENSED_EVENTS_FLUSH_PERIOD:
                        _thread_data.condensed_events_flush_sec = sec

                        for key in getattr(_thread_data, "condensed_events", []):
                            condensed = False
                            events = _thread_data.condensed_events[key]

                            first_event = events[0]
                            condensed_event = [_ for _ in first_event]

                            for i in xrange(1, len(events)):
                                current_event = events[i]
                                for j in xrange(3, 7):  # src_port, dst_ip, dst_port, proto
                                    if current_event[j] != condensed_event[j]:
                                        condensed = True
                                        if not isinstance(condensed_event[j], set):
                                            condensed_event[j] = set((condensed_event[j],))
                                        condensed_event[j].add(current_event[j])

                            if condensed:
                                for i in xrange(len(condensed_event)):
                                    if isinstance(condensed_event[i], set):
                                        condensed_event[i] = ','.join(str(_) for _ in sorted(condensed_event[i]))

                            log_event(condensed_event, skip_condensing=True)

                        _thread_data.condensed_events = {}

                    if any(_ in info for _ in CONDENSE_ON_TRAIL_KEYWORDS):
                        if not hasattr(_thread_data, "condensed_events"):
                            _thread_data.condensed_events = {}
                        key = (src_ip, trail)
                        if key not in _thread_data.condensed_events:
                            _thread_data.condensed_events[key] = []
                        _thread_data.condensed_events[key].append(event_tuple)
{'fraction_non_alphanumeric': 0.09521218715995647, 'fraction_numerical': 0.0045701849836779105, 'mean_word_length': 2.0636666666666668, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4346938', 'n_tokens_mistral': 2782, 'n_tokens_neox': 2683, 'n_words': 559}