<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

VERSION = "0.6.8"
VERDATE = "3 Jan 2012"

import os, os.path, math, fnmatch, re, time, calendar, string, socket, sys, random, ConfigParser, socket, urllib
from copy import copy
from hashlib import md5 as md5

try:
    set
except NameError:
    from sets import Set as set

geocoder = None
try:
    import GeoIP
    geocoder = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
except:
    pass

LOG_LEVEL          = 1        # 0 == quiet, 1 == normal, 2 == debug
DISC_SYNC_CNT      = 100000   # number of records to hold in memory before flushing to disk
SORT_BUFFER_LENGTH = 100000   # number of records to hold before triggering a sort & prune operation
PROGRESS_INTERVAL  = 20000    # ie warn('processed X lines...')

# randomize the disc sync and sort_buffer len to reduce the
# chance of thrashing the disks in mutlicore mode.
DISC_SYNC_CNT      += (DISC_SYNC_CNT * 0.08 * random.random())
SORT_BUFFER_LENGTH += (SORT_BUFFER_LENGTH * 0.08 * random.random())


def warn(s):
    if LOG_LEVEL < 1: return
    sys.stderr.write(s+'\n')


# change these defaults in your wtop.cfg file
LOG_ROOT = LOG_FILE = LOG_FILE_TYPE = LOG_FORMAT = DEFAULT_OUTPUT_FIELDS = MAX_REQUEST_TIME = MIN_RPS = re_robots =re_generic = re_classes = None
config = {}


def cfg_home():
    if (os.name != 'posix'): return sys.prefix
    return '/etc'

# yes, this is ugly.
def configure(cfg_file=None):
    global config, LOG_ROOT, LOG_FILE, LOG_FILE_TYPE, LOG_FORMAT, DEFAULT_OUTPUT_FIELDS, MAX_REQUEST_TIME, MIN_RPS,re_robots,re_generic,re_classes,LOG_PATTERN,LOG_COLUMNS

    if not cfg_file:
        cfg_file = os.path.join(cfg_home(), 'wtop.cfg')

    if not os.path.exists(cfg_file):
        raise Exception("can't read config file: '%s'" % cfg_file)

    config = ConfigParser.ConfigParser()
    config.read(cfg_file)

    LOG_FILE_TYPE          = config.get('main', 'log_file_type')

    LOG_ROOT               = config.get('main', 'log_root')
    LOG_FORMAT             = config.get('main', 'log_format')
    DEFAULT_OUTPUT_FIELDS  = config.get('main', 'default_output_fields').split(',')
    MAX_REQUEST_TIME       = int(config.get('wtop', 'max_request_time'))
    MIN_RPS                = float(config.get('wtop', 'min_rps'))
    classes = [(o, config.get('classes', o)) for o in config.options('classes')]

    ## compile a godawful bunch of regexps
    re_robots  = re.compile(config.get('patterns', 'robots'), re.I)
    re_generic = re.compile(config.get('patterns', 'generic'))
    re_classes = [(x[0], re.compile(x[1], re.I)) for x in classes]

    if LOG_FILE_TYPE == 'apache':
        # these may be overridden later by the logrep command line program because it
        # knows the fields the user asked for.
        LOG_FILE                = config.get('main', 'log_file')
        LOG_PATTERN,LOG_COLUMNS = format2regexp(config.get('main', 'log_format'))


def flatten(x):
    result = []
    for el in x:
        if hasattr(el, "__iter__") and not isinstance(el, basestring):
            result.extend(flatten(el))
        else:
            result.append(el)
    return result



# translate log format string into a column list and regexp
restr            = r'(\S+)'
restr_skipped    = r'\S+'
requoted         = r'([^"]*)'
requoted_skipped = r'[^"]*'

# {DIRECTIVE_SYMBOL: (FIELD_NAME, REGEX_WHEN_NEEDED, REGEX_WHEN_SKIPPED)}
LOG_DIRECTIVES = {
    'h' : ('ip',         restr, restr_skipped),  # ip is an odd one, since the default Apache is %h but
    'a' : ('ip',         restr, restr_skipped),  # HostnameLookups changes its content and %a is ALSO the ip. bleh.
    'A' : ('lip',        restr, restr_skipped),  # server (local) IP
    'l' : ('auth',       restr, restr_skipped),
    'u' : ('username',   restr, restr_skipped),
    't' : ('timestamp',  r'\[?(\S+(?:\s+[\-\+]\d+)?)\]?', r'\[?\S+(?:\s+[\-\+]\d+)?\]?'),
    'r' : ('request',    r'(\S+ \S+ \S+)', r'\S+ \S+ \S+'),
    'm' : ('method',     restr, restr_skipped),
    'D' : ('msec',       restr, restr_skipped),
    'F' : ('fbmsec',     restr, restr_skipped),
    'q' : ('query',      restr, restr_skipped),
    's' : ('status',     restr, restr_skipped),
    'b' : ('bytes',      restr, restr_skipped),
    'B' : ('bytes',      restr, restr_skipped), # NB: may change to 'bytes_out'
    'I' : ('bytes_in',   restr, restr_skipped),
    'v' : ('domain',     restr, restr_skipped), # Host header
    'V' : ('domain',     restr, restr_skipped), # actual vhost. May clobber %v
    'p' : ('port',       restr, restr_skipped),
    '{ratio}n' : ('ratio', requoted, requoted_skipped), #todo: need generic %{foo}X parsing?
    '{host}i' : ('host', requoted, requoted_skipped),
    '{referer}i' : ('ref', requoted, requoted_skipped),
    '{user-agent}i' : ('ua', requoted, requoted_skipped),
    'ignore' : ('ignore', restr, restr_skipped),
    'ignorequot' : ('ignore', requoted, requoted_skipped),
}

# lower-case some apache header options.
def lcase(s):
    if s.find('{') > -1:
        return s.lower()
    return s

# given an Apache LogFormat string and an optional list of relevant fields,
# returns a regular expression that will parse the equivalent log line and
# extract the necessary fields.
def format2regexp(fmt, relevant_fields=()):
    red = re.compile('%[\>\<\!,\d]*([\}\{\w\-]+)')
    directives = map(lcase, red.findall(fmt))
    colnames = []
    pat = fmt
    for k in directives:
        if not LOG_DIRECTIVES.has_key(k) and not k.find('{') > -1:
            continue

        field, pattern, skip_pattern = LOG_DIRECTIVES[k]
        atom = pattern
        if (not relevant_fields) or (field in relevant_fields):
            colnames.append(field)
        else:
            atom = skip_pattern

        if k.find('{') > -1:
            p = re.compile('%[\>\<\!,\d]*'+k.replace('}', '.').replace('{', '.'), re.I)
            pat = p.sub(atom, pat, re.I)
        else:
            pat = re.sub('%[\>\<\!,\d]*'+k, atom, pat)

    leftover = red.findall(pat)
    if leftover:
        warn('unrecognized format directives: %%%s' % ', %'.join(leftover))

    return pat, flatten(colnames)


# NCSA log format is whimsical. often a 0 is printed as '-'
def safeint(s):
    return int(s.replace('-', '0'))

### timestamp parsing
## "...the %z escape that expands to the preferred hour/minute
##     offset is not supported by all ANSI C libraries..."
## http://docs.python.org/library/time.html
## GRRRR.
def tz2secs(s):
  plusminus = 1 if s[0] == '-' else -1
  return ((int(s[1:3])*3600) + (int(s[3:5]))*60) * plusminus

# 21/Jul/2008:18:09:00 -0700   -->   1216688940
# 03/Jan/2012:12:11:24 +0000   -->   1325592684
def apache2unixtime(t):
    return calendar.timegm(time.strptime(t[:20]+' GMT', '%d/%b/%Y:%H:%M:%S %Z')) + tz2secs(t[21:26])

# 21/Jul/2008:18:09:00 -0700   -->   (2008, 7, 21, 18, 9)
def apache2dateparts(t):
    return int(t[7:11]), months[t[3:6]], int(t[0:2]), int(t[12:14]), int(t[15:17])


# keeps a count of seen remote IP addresses. returns
# a value for the ipcnt field.
# nb: possbile memory problem with > 10M records
# derp. IPv6?
reip = re.compile(r'^\d+\.\d+\.\d+\.\d+$')
ipcnts = {}
def count_ips(ip):
    # compact the ip to 4 bytes for the ipcnts table
    ipkey = ip
    if reip.match(ip): # hack: some people still have HostnameLookups on
        ipkey = socket.inet_aton(ip)
    ipcnts[ipkey] = ipcnts.get(ipkey, 0) + 1
    return ipcnts[ipkey]<commit_msg>fix month handling in apache2dateparts<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

VERSION = "0.6.8"
VERDATE = "3 Jan 2012"

import os, os.path, math, fnmatch, re, time, calendar, string, socket, sys, random, ConfigParser, socket, urllib
from copy import copy
from hashlib import md5 as md5

try:
    set
except NameError:
    from sets import Set as set

geocoder = None
try:
    import GeoIP
    geocoder = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE)
except:
    pass

LOG_LEVEL          = 1        # 0 == quiet, 1 == normal, 2 == debug
DISC_SYNC_CNT      = 100000   # number of records to hold in memory before flushing to disk
SORT_BUFFER_LENGTH = 100000   # number of records to hold before triggering a sort & prune operation
PROGRESS_INTERVAL  = 20000    # ie warn('processed X lines...')

# randomize the disc sync and sort_buffer len to reduce the
# chance of thrashing the disks in mutlicore mode.
DISC_SYNC_CNT      += (DISC_SYNC_CNT * 0.08 * random.random())
SORT_BUFFER_LENGTH += (SORT_BUFFER_LENGTH * 0.08 * random.random())


def warn(s):
    if LOG_LEVEL < 1: return
    sys.stderr.write(s+'\n')


# change these defaults in your wtop.cfg file
LOG_ROOT = LOG_FILE = LOG_FILE_TYPE = LOG_FORMAT = DEFAULT_OUTPUT_FIELDS = MAX_REQUEST_TIME = MIN_RPS = re_robots =re_generic = re_classes = None
config = {}


def cfg_home():
    if (os.name != 'posix'): return sys.prefix
    return '/etc'

# yes, this is ugly.
def configure(cfg_file=None):
    global config, LOG_ROOT, LOG_FILE, LOG_FILE_TYPE, LOG_FORMAT, DEFAULT_OUTPUT_FIELDS, MAX_REQUEST_TIME, MIN_RPS,re_robots,re_generic,re_classes,LOG_PATTERN,LOG_COLUMNS

    if not cfg_file:
        cfg_file = os.path.join(cfg_home(), 'wtop.cfg')

    if not os.path.exists(cfg_file):
        raise Exception("can't read config file: '%s'" % cfg_file)

    config = ConfigParser.ConfigParser()
    config.read(cfg_file)

    LOG_FILE_TYPE          = config.get('main', 'log_file_type')

    LOG_ROOT               = config.get('main', 'log_root')
    LOG_FORMAT             = config.get('main', 'log_format')
    DEFAULT_OUTPUT_FIELDS  = config.get('main', 'default_output_fields').split(',')
    MAX_REQUEST_TIME       = int(config.get('wtop', 'max_request_time'))
    MIN_RPS                = float(config.get('wtop', 'min_rps'))
    classes = [(o, config.get('classes', o)) for o in config.options('classes')]

    ## compile a godawful bunch of regexps
    re_robots  = re.compile(config.get('patterns', 'robots'), re.I)
    re_generic = re.compile(config.get('patterns', 'generic'))
    re_classes = [(x[0], re.compile(x[1], re.I)) for x in classes]

    if LOG_FILE_TYPE == 'apache':
        # these may be overridden later by the logrep command line program because it
        # knows the fields the user asked for.
        LOG_FILE                = config.get('main', 'log_file')
        LOG_PATTERN,LOG_COLUMNS = format2regexp(config.get('main', 'log_format'))


def flatten(x):
    result = []
    for el in x:
        if hasattr(el, "__iter__") and not isinstance(el, basestring):
            result.extend(flatten(el))
        else:
            result.append(el)
    return result



# translate log format string into a column list and regexp
restr            = r'(\S+)'
restr_skipped    = r'\S+'
requoted         = r'([^"]*)'
requoted_skipped = r'[^"]*'

# {DIRECTIVE_SYMBOL: (FIELD_NAME, REGEX_WHEN_NEEDED, REGEX_WHEN_SKIPPED)}
LOG_DIRECTIVES = {
    'h' : ('ip',         restr, restr_skipped),  # ip is an odd one, since the default Apache is %h but
    'a' : ('ip',         restr, restr_skipped),  # HostnameLookups changes its content and %a is ALSO the ip. bleh.
    'A' : ('lip',        restr, restr_skipped),  # server (local) IP
    'l' : ('auth',       restr, restr_skipped),
    'u' : ('username',   restr, restr_skipped),
    't' : ('timestamp',  r'\[?(\S+(?:\s+[\-\+]\d+)?)\]?', r'\[?\S+(?:\s+[\-\+]\d+)?\]?'),
    'r' : ('request',    r'(\S+ \S+ \S+)', r'\S+ \S+ \S+'),
    'm' : ('method',     restr, restr_skipped),
    'D' : ('msec',       restr, restr_skipped),
    'F' : ('fbmsec',     restr, restr_skipped),
    'q' : ('query',      restr, restr_skipped),
    's' : ('status',     restr, restr_skipped),
    'b' : ('bytes',      restr, restr_skipped),
    'B' : ('bytes',      restr, restr_skipped), # NB: may change to 'bytes_out'
    'I' : ('bytes_in',   restr, restr_skipped),
    'v' : ('domain',     restr, restr_skipped), # Host header
    'V' : ('domain',     restr, restr_skipped), # actual vhost. May clobber %v
    'p' : ('port',       restr, restr_skipped),
    '{ratio}n' : ('ratio', requoted, requoted_skipped), #todo: need generic %{foo}X parsing?
    '{host}i' : ('host', requoted, requoted_skipped),
    '{referer}i' : ('ref', requoted, requoted_skipped),
    '{user-agent}i' : ('ua', requoted, requoted_skipped),
    'ignore' : ('ignore', restr, restr_skipped),
    'ignorequot' : ('ignore', requoted, requoted_skipped),
}

# lower-case some apache header options.
def lcase(s):
    if s.find('{') > -1:
        return s.lower()
    return s

# given an Apache LogFormat string and an optional list of relevant fields,
# returns a regular expression that will parse the equivalent log line and
# extract the necessary fields.
def format2regexp(fmt, relevant_fields=()):
    red = re.compile('%[\>\<\!,\d]*([\}\{\w\-]+)')
    directives = map(lcase, red.findall(fmt))
    colnames = []
    pat = fmt
    for k in directives:
        if not LOG_DIRECTIVES.has_key(k) and not k.find('{') > -1:
            continue

        field, pattern, skip_pattern = LOG_DIRECTIVES[k]
        atom = pattern
        if (not relevant_fields) or (field in relevant_fields):
            colnames.append(field)
        else:
            atom = skip_pattern

        if k.find('{') > -1:
            p = re.compile('%[\>\<\!,\d]*'+k.replace('}', '.').replace('{', '.'), re.I)
            pat = p.sub(atom, pat, re.I)
        else:
            pat = re.sub('%[\>\<\!,\d]*'+k, atom, pat)

    leftover = red.findall(pat)
    if leftover:
        warn('unrecognized format directives: %%%s' % ', %'.join(leftover))

    return pat, flatten(colnames)


# NCSA log format is whimsical. often a 0 is printed as '-'
def safeint(s):
    return int(s.replace('-', '0'))

### timestamp parsing
## "...the %z escape that expands to the preferred hour/minute
##     offset is not supported by all ANSI C libraries..."
## http://docs.python.org/library/time.html
## GRRRR.
def tz2secs(s):
  plusminus = 1 if s[0] == '-' else -1
  return ((int(s[1:3])*3600) + (int(s[3:5]))*60) * plusminus

# 21/Jul/2008:18:09:00 -0700   -->   1216688940
# 03/Jan/2012:12:11:24 +0000   -->   1325592684
def apache2unixtime(t):
    return calendar.timegm(time.strptime(t[:20]+' GMT', '%d/%b/%Y:%H:%M:%S %Z')) + tz2secs(t[21:26])

# 21/Jul/2008:18:09:00 -0700   -->   (2008, 7, 21, 18, 9)
def apache2dateparts(t):
    return int(t[7:11]), time.strptime(t[3:6], "%b").tm_mon, int(t[0:2]), int(t[12:14]), int(t[15:17])


# keeps a count of seen remote IP addresses. returns
# a value for the ipcnt field.
# nb: possbile memory problem with > 10M records
# derp. IPv6?
reip = re.compile(r'^\d+\.\d+\.\d+\.\d+$')
ipcnts = {}
def count_ips(ip):
    # compact the ip to 4 bytes for the ipcnts table
    ipkey = ip
    if reip.match(ip): # hack: some people still have HostnameLookups on
        ipkey = socket.inet_aton(ip)
    ipcnts[ipkey] = ipcnts.get(ipkey, 0) + 1
    return ipcnts[ipkey]
{'fraction_non_alphanumeric': 0.1625008421478138, 'fraction_numerical': 0.022973792360035033, 'mean_word_length': 3.0380848748639826, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '697797', 'n_tokens_mistral': 5541, 'n_tokens_neox': 5289, 'n_words': 1396}