<commit_before>#!/usr/bin/env python
 
## Tiny Syslog Server in Python.
## from a rad gist at https://gist.github.com/marcelom/4218010
## This is a tiny syslog server that is able to receive UDP based syslog
## entries on a specified port and save them to a file.
## That's it... it does nothing else...
## There are a few configuration parameters.
 
LOG_FILE = 'remote_aka.log'
HOST, PORT = "192.168.1.2", 514

# import logging
import SocketServer
import beanstalkc
from urlparse import urlparse as parse
from collections import Counter

nixList = ['png','jpeg','jpg','css','js','ipa','ico','gif','mov','mp4','svg','json','woff','woff2','pdf','mp3','crl','webp','jsonp'
'PNG','JPEG','JPG','CSS','JS','IPA','ICO','GIF','MOV','MP4','SVG','JSON','WOFF','WOFF2','PDF','MP3','CRL','WEBP','JSONP']
local_host = '192.168.1.1'
cnt = Counter()

def isValid(line_in):
    # check for jpeg, jpg, gif, js, etc
    # return True if it's a valid url
    if 'Host'==line_in:
        return False
    try:
        p = parse(line_in)
        for n in nixList:
            if p.path.endswith(n):
                return False
            if local_host in p.netloc:
                return False
            if 'gravatar.com' in p.netloc: # this domain is just nasty, all infinite redirects and fury, signifying nothing
                return False
        cnt[line_in] += 1
        if cnt[line_in] >1:
            return False
        return True
    except:
        return False

# logging.basicConfig(level=logging.INFO, format='%(message)s', datefmt='', filename=LOG_FILE, filemode='a')
 
class SyslogUDPHandler(SocketServer.BaseRequestHandler):<commit_msg>really was hoping to avoid ringbuffer in python<commit_after>#!/usr/bin/env python
 
## Tiny Syslog Server in Python.
## from a rad gist at https://gist.github.com/marcelom/4218010
## This is a tiny syslog server that is able to receive UDP based syslog
## entries on a specified port and save them to a file.
## That's it... it does nothing else...
## There are a few configuration parameters.
 
LOG_FILE = 'remote_aka.log'
HOST, PORT = "192.168.1.2", 514

# import logging
import SocketServer
import beanstalkc
from urlparse import urlparse as parse
from collections import Counter

nixList = ['png','jpeg','jpg','css','js','ipa','ico','gif','mov','mp4','svg','json','woff','woff2','pdf','mp3','crl','webp','jsonp'
'PNG','JPEG','JPG','CSS','JS','IPA','ICO','GIF','MOV','MP4','SVG','JSON','WOFF','WOFF2','PDF','MP3','CRL','WEBP','JSONP']
local_host = '192.168.1.1'
cnt = Counter()

def isValid(line_in):
    # check for jpeg, jpg, gif, js, etc
    # return True if it's a valid url
    if 'Host'==line_in:
        return False
    if 'GET'==line_in:
        return False
    try:
        p = parse(line_in)
        for n in nixList:
            if p.path.endswith(n):
                return False
            if local_host in p.netloc:
                return False
            if 'gravatar.com' in p.netloc: # this domain is just nasty, all infinite redirects and fury, signifying nothing
                return False
        cnt[line_in] += 1
        if cnt[line_in] >1:
            return False
        return True
    except:
        return False

# logging.basicConfig(level=logging.INFO, format='%(message)s', datefmt='', filename=LOG_FILE, filemode='a')
 
class SyslogUDPHandler(SocketServer.BaseRequestHandler):
{'fraction_non_alphanumeric': 0.14732142857142858, 'fraction_numerical': 0.02023809523809524, 'mean_word_length': 3.2329974811083124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2297376', 'n_tokens_mistral': 1139, 'n_tokens_neox': 1063, 'n_words': 316}