<commit_before>#!/usr/bin/python3
# -*- coding: utf-8
##############################################################################
# Politraf, check connections for network IOC
##############################################################################


import time
import datetime
import logging

from pytz import timezone
import yaml

import dbmodels


# Set logging level
logging.basicConfig(level = logging.INFO)


# Read config
with open("/etc/politraf/config.yaml", 'r') as stream:
    try:
        config = (yaml.load(stream))
        time_zone = config['time_zone']
        tz = timezone(time_zone)
        url = config['db_url']
        name = config['username']
        passw = config['password']
    except yaml.YAMLError as e:
        logging.error("Error.",e)
    logging.info("Config is OK")


# Init clickhouse
try:
    db = dbmodels.Database('politraf', db_url=url, username=name, password=<PASSWORD>, readonly=False, autocreate=True)
except Exception as e:
    logging.error("Error.",e)


# 5 min time
to_time = datetime.datetime.now(tz)
from_time = (datetime.datetime.now(tz) - datetime.timedelta(minutes=1)).replace(microsecond=0)
from_time_epoch = str(from_time.timestamp())
today = datetime.datetime.strftime(datetime.datetime.now(tz), '%Y-%m-%d')

def get_traf_last():
        try:
            start = time.time()
            logging.info("Starting fetch traffic stat ...")
            for row in db.select('SELECT uniq(dst_addr), * FROM politraf.connstats WHERE timestamp >= toDateTime('+from_time_epoch+') GROUP BY dst_addr, event_date, timestamp, protocol, src_addr, src_port, dst_port, qry_name ORDER BY timestamp'):
                timestamp = datetime.datetime.now(tz)
                if not row.qry_name == 'none':
                    for ioc in db.select('SELECT * FROM politraf.ioc_otx WHERE indicator = \''+row.qry_name+'\' ORDER BY timestamp'):
                        db.insert([
                        dbmodels.IOCStats(event_date=today, timestamp=timestamp, protocol=row.protocol, src_addr=row.src_addr, src_port=row.src_port, dst_addr=row.dst_addr, dst_port=row.dst_port, qry_name=row.qry_name, indicator=ioc.indicator, name=ioc.name, references=ioc.references)
                        ])    
                else:
                    for ioc in db.select('SELECT * FROM politraf.ioc_otx WHERE indicator = \''+row.dst_addr+'\' ORDER BY timestamp'):
                        db.insert([
                        dbmodels.IOCStats(event_date=today, timestamp=timestamp, protocol=row.protocol, src_addr=row.src_addr, src_port=row.src_port, dst_addr=row.dst_addr, dst_port=row.dst_port, qry_name=row.qry_name, indicator=ioc.indicator, name=ioc.name, references=ioc.references)
                        ])    
            end = time.time()
            time_to_end = end - start
            print('Time to all events ', time_to_end)
        except Exception as e:
            logging.error("Error.",e)


def main():<commit_msg>	change period to 10m:   iocwatch.py<commit_after>#!/usr/bin/python3
# -*- coding: utf-8
##############################################################################
# Politraf, check connections for network IOC
##############################################################################


import time
import datetime
import logging

from pytz import timezone
import yaml

import dbmodels


# Set logging level
logging.basicConfig(level = logging.DEBUG)


# Read config
with open("/etc/politraf/config.yaml", 'r') as stream:
    try:
        config = (yaml.load(stream))
        time_zone = config['time_zone']
        tz = timezone(time_zone)
        url = config['db_url']
        name = config['username']
        passw = config['password']
    except yaml.YAMLError as e:
        logging.error("Error.",e)
    logging.info("Config is OK")


# Init clickhouse
try:
    db = dbmodels.Database('politraf', db_url=url, username=name, password=<PASSWORD>, readonly=False, autocreate=True)
except Exception as e:
    logging.error("Error.",e)


# 5 min time
to_time = datetime.datetime.now(tz)
from_time = (datetime.datetime.now(tz) - datetime.timedelta(minutes=10)).replace(microsecond=0)
from_time_epoch = str(from_time.timestamp())
today = datetime.datetime.strftime(datetime.datetime.now(tz), '%Y-%m-%d')

def get_traf_last():
        try:
            start = time.time()
            logging.info("Starting fetch traffic stat ...")
            for row in db.select('SELECT uniq(dst_addr), * FROM politraf.connstats WHERE timestamp >= toDateTime('+from_time_epoch+') GROUP BY dst_addr, event_date, timestamp, protocol, src_addr, src_port, dst_port, qry_name ORDER BY timestamp'):
                timestamp = datetime.datetime.now(tz)
                if not row.qry_name == 'none' and "arp" not in row.qry_name:
                    logging.info(row.qry_name)
                    for ioc in db.select('SELECT * FROM politraf.ioc_otx WHERE indicator = \''+row.qry_name+'\' ORDER BY timestamp'):
                        db.insert([
                        dbmodels.IOCStats(event_date=today, timestamp=timestamp, protocol=row.protocol, src_addr=row.src_addr, src_port=row.src_port, dst_addr=row.dst_addr, dst_port=row.dst_port, qry_name=row.qry_name, indicator=ioc.indicator, name=ioc.name, references=ioc.references)
                        ])    
                else:
                    logging.info(row.dst_addr)
                    for ioc in db.select('SELECT * FROM politraf.ioc_otx WHERE indicator = \''+row.dst_addr+'\' ORDER BY timestamp'):
                        db.insert([
                        dbmodels.IOCStats(event_date=today, timestamp=timestamp, protocol=row.protocol, src_addr=row.src_addr, src_port=row.src_port, dst_addr=row.dst_addr, dst_port=row.dst_port, qry_name=row.qry_name, indicator=ioc.indicator, name=ioc.name, references=ioc.references)
                        ])    
            end = time.time()
            time_to_end = end - start
            print('Time to all events ', time_to_end)
        except Exception as e:
            logging.error("Error.",e)


def main():
{'fraction_non_alphanumeric': 0.1654865799440145, 'fraction_numerical': 0.002140622427136506, 'mean_word_length': 2.9621656881930853, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4636624', 'n_tokens_mistral': 1836, 'n_tokens_neox': 1764, 'n_words': 387}