<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2014-2015 <NAME> (@stamparm)
See the file 'LICENSE' for copying permission
"""

import mmap
import os
import socket
import subprocess
import struct
import sys
import threading
import time
import traceback

sys.dont_write_bytecode = True

from core.common import check_sudo
from core.common import load_trails
from core.enums import BLOCK_MARKER
from core.enums import TRAIL
from core.log import create_log_directory
from core.log import log_event
from core.parallel import worker
from core.parallel import write_block
from core.settings import BUFFER_LENGTH
from core.settings import config
from core.settings import ETH_LENGTH
from core.settings import IPPROTO
from core.settings import IPPROTO_LUT
from core.settings import NO_SUCH_NAME_PER_HOUR_THRESHOLD
from core.settings import NO_SUCH_NAME_COUNTERS
from core.settings import REGULAR_SLEEP_TIME
from core.settings import SNAP_LEN
from core.settings import SUSPICIOUS_DOMAIN_LENGTH_THRESHOLD
from core.settings import trails
from core.settings import WHITELIST
from core.update import update

_buffer = None
_cap = None
_count = 0
_multiprocessing = False
_n = None
_datalink = None

if config.USE_MULTIPROCESSING:
    try:
        import multiprocessing

        if multiprocessing.cpu_count() > 1:
            _multiprocessing = True
    except (ImportError, OSError, NotImplementedError):
        pass

try:
    import pcapy
except ImportError:
    if subprocess.mswindows:
        exit("[!] please install Pcapy (e.g. 'https://breakingcode.wordpress.com/?s=pcapy') and WinPcap (e.g. 'http://www.winpcap.org/install/')")
    else:
        exit("[!] please install Pcapy (e.g. 'apt-get install python-pcapy')")

def _process_packet(packet, sec, usec):
    """
    Processes single (raw) packet
    """

    try:
        if _datalink == pcapy.DLT_LINUX_SLL:
            packet = packet[2:]

        eth_header = struct.unpack("!HH8sH", packet[:ETH_LENGTH])
        eth_protocol = socket.ntohs(eth_header[3])

        if eth_protocol == IPPROTO:  # IP
            ip_header = struct.unpack("!BBHHHBBH4s4s", packet[ETH_LENGTH:ETH_LENGTH + 20])
            ip_length = ip_header[2]
            packet = packet[:ETH_LENGTH + ip_length]  # truncate
            iph_length = (ip_header[0] & 0xF) << 2
            protocol = ip_header[6]
            src_ip = socket.inet_ntoa(ip_header[8])
            dst_ip = socket.inet_ntoa(ip_header[9])

            if protocol == socket.IPPROTO_TCP:  # TCP
                i = iph_length + ETH_LENGTH
                src_port, dst_port, _, _, doff_reserved, flags = struct.unpack("!HHLLBB", packet[i:i+14])

                if flags == 2:  # SYN set (only)
                    if dst_ip in trails[TRAIL.IP]:
                        log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.IP, dst_ip, trails[TRAIL.IP][dst_ip][0], trails[TRAIL.IP][dst_ip][1]))
                    elif src_ip in trails[TRAIL.IP]:
                        log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.IP, src_ip, trails[TRAIL.IP][src_ip][0], trails[TRAIL.IP][src_ip][1]))

                if flags & 8 != 0:  # PSH set
                    tcph_length = doff_reserved >> 4
                    h_size = ETH_LENGTH + iph_length + (tcph_length << 2)
                    data = packet[h_size:]

                    if dst_port == 80 and len(data) > 0:
                        index = data.find("\r\n")
                        if index >= 0:
                            line = data[:index]
                            if line.count(' ') == 2 and " HTTP/" in line:
                                path = line.split(' ')[1]
                            else:
                                return
                        else:
                            return

                        index = data.find("\r\nHost:")
                        if index >= 0:
                            index = index + len("\r\nHost:")
                            host = data[index:data.find("\r\n", index)]
                            host = host.strip()
                        else:
                            return

                        path = path.split('?')[0]
                        path = path.rstrip('/')
                        paths = [path]

                        _ = os.path.splitext(paths[-1])
                        if _[1]:
                            paths.append(_[0])

                        if paths[-1].count('/') > 1:
                            paths.append(paths[-1][:paths[-1].rfind('/')])

                        for path in paths:
                            if path and path in trails[TRAIL.URL]:
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.URL, path, trails[TRAIL.URL][path][0], trails[TRAIL.URL][path][1]))
                                break

                            url = "%s%s" % (host, path)
                            if url in trails[TRAIL.URL]:
                                log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.URL, url, trails[TRAIL.URL][url][0], trails[TRAIL.URL][url][1]))
                                break
<commit_msg>Adding more info into URL events<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2014-2015 <NAME> (@stamparm)
See the file 'LICENSE' for copying permission
"""

import mmap
import os
import socket
import subprocess
import struct
import sys
import threading
import time
import traceback

sys.dont_write_bytecode = True

from core.common import check_sudo
from core.common import load_trails
from core.enums import BLOCK_MARKER
from core.enums import TRAIL
from core.log import create_log_directory
from core.log import log_event
from core.parallel import worker
from core.parallel import write_block
from core.settings import BUFFER_LENGTH
from core.settings import config
from core.settings import ETH_LENGTH
from core.settings import IPPROTO
from core.settings import IPPROTO_LUT
from core.settings import NO_SUCH_NAME_PER_HOUR_THRESHOLD
from core.settings import NO_SUCH_NAME_COUNTERS
from core.settings import REGULAR_SLEEP_TIME
from core.settings import SNAP_LEN
from core.settings import SUSPICIOUS_DOMAIN_LENGTH_THRESHOLD
from core.settings import trails
from core.settings import WHITELIST
from core.update import update

_buffer = None
_cap = None
_count = 0
_multiprocessing = False
_n = None
_datalink = None

if config.USE_MULTIPROCESSING:
    try:
        import multiprocessing

        if multiprocessing.cpu_count() > 1:
            _multiprocessing = True
    except (ImportError, OSError, NotImplementedError):
        pass

try:
    import pcapy
except ImportError:
    if subprocess.mswindows:
        exit("[!] please install Pcapy (e.g. 'https://breakingcode.wordpress.com/?s=pcapy') and WinPcap (e.g. 'http://www.winpcap.org/install/')")
    else:
        exit("[!] please install Pcapy (e.g. 'apt-get install python-pcapy')")

def _process_packet(packet, sec, usec):
    """
    Processes single (raw) packet
    """

    try:
        if _datalink == pcapy.DLT_LINUX_SLL:
            packet = packet[2:]

        eth_header = struct.unpack("!HH8sH", packet[:ETH_LENGTH])
        eth_protocol = socket.ntohs(eth_header[3])

        if eth_protocol == IPPROTO:  # IP
            ip_header = struct.unpack("!BBHHHBBH4s4s", packet[ETH_LENGTH:ETH_LENGTH + 20])
            ip_length = ip_header[2]
            packet = packet[:ETH_LENGTH + ip_length]  # truncate
            iph_length = (ip_header[0] & 0xF) << 2
            protocol = ip_header[6]
            src_ip = socket.inet_ntoa(ip_header[8])
            dst_ip = socket.inet_ntoa(ip_header[9])

            if protocol == socket.IPPROTO_TCP:  # TCP
                i = iph_length + ETH_LENGTH
                src_port, dst_port, _, _, doff_reserved, flags = struct.unpack("!HHLLBB", packet[i:i+14])

                if flags == 2:  # SYN set (only)
                    if dst_ip in trails[TRAIL.IP]:
                        log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.IP, dst_ip, trails[TRAIL.IP][dst_ip][0], trails[TRAIL.IP][dst_ip][1]))
                    elif src_ip in trails[TRAIL.IP]:
                        log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.IP, src_ip, trails[TRAIL.IP][src_ip][0], trails[TRAIL.IP][src_ip][1]))

                if flags & 8 != 0:  # PSH set
                    tcph_length = doff_reserved >> 4
                    h_size = ETH_LENGTH + iph_length + (tcph_length << 2)
                    data = packet[h_size:]

                    if dst_port == 80 and len(data) > 0:
                        index = data.find("\r\n")
                        if index >= 0:
                            line = data[:index]
                            if line.count(' ') == 2 and " HTTP/" in line:
                                path = line.split(' ')[1]
                            else:
                                return
                        else:
                            return

                        index = data.find("\r\nHost:")
                        if index >= 0:
                            index = index + len("\r\nHost:")
                            host = data[index:data.find("\r\n", index)]
                            host = host.strip()
                        else:
                            return

                        url = "%s%s" % (host, path)

                        checks = [path.rstrip('/')]
                        if '?' in path:
                            checks.append(path.split('?')[0].rstrip('/'))

                        _ = os.path.splitext(checks[-1])
                        if _[1]:
                            checks.append(_[0])

                        if checks[-1].count('/') > 1:
                            checks.append(checks[-1][:checks[-1].rfind('/')])

                        for check in filter(None, checks):
                            for _ in ("", host):
                                check = "%s%s" % (_, check)
                                if check in trails[TRAIL.URL]:
                                    parts = url.split(check)
                                    other = ("(%s)" % _ for _ in parts)
                                    trail = check.join(other)
                                    log_event((sec, usec, src_ip, src_port, dst_ip, dst_port, "TCP", TRAIL.URL, trail, trails[TRAIL.URL][check][0], trails[TRAIL.URL][check][1]))
                                    break

{'fraction_non_alphanumeric': 0.10551763367463027, 'fraction_numerical': 0.010049298445202881, 'mean_word_length': 1.6931324993617565, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5947379', 'n_tokens_mistral': 3235, 'n_tokens_neox': 3051, 'n_words': 757}