<commit_before># -*- coding: utf-8 -*-

'''nginx log line plugin
'''

from __future__ import (
    absolute_import,
    division,
    print_function,
    unicode_literals,
    with_statement,
)

import logging

from metricol.commons import decode_time


LOG = logging.getLogger(__name__)


def clean_uri(uri):
    '''Cleans URI
    '''
    if uri.count('/') > 1:
        prefix = uri.split('/', 1)[0]
        if prefix.isalnum():
            uri = prefix.replace('.', '_')
        else:
            uri = ''
    if not uri:
        uri = '_other'

    return uri


def parse_log_lines(lines, pattern_fn):
    '''Parses log line using pattern
    '''
    for idx, line in enumerate(lines):
        match = pattern_fn(line)
        if not match:
            continue

        data = match.groupdict()
        if 'time' in data:
            data['time'] = decode_time(data['time'])
        if 'uri' in data:
            data['uri'] = clean_uri(data['uri'])
        if 'http' in data:
            data['http'] = data['http'].replace('.', '_')
        if 'method' in data:
            data['method.' + data.pop('method')] = 1

        for field in ['uctim', 'uhtim', 'urtim', 'gzip']:
            if field not in data:
                continue
            if data[field] == '-':
                del data[field]
            else:
                data[field] = float(data[field])

        if 'pipe' in data:
            if data['pipe'] != 'p':
                del data['pipe']

        LOG.debug('DATA: %s', repr(data))

        time = data.pop('time')
        if time:
            yield (idx, (time, data))<commit_msg>statsite metrics fix tryout<commit_after># -*- coding: utf-8 -*-

'''nginx log line plugin
'''

from __future__ import (
    absolute_import,
    division,
    print_function,
    unicode_literals,
    with_statement,
)

import logging

from metricol.commons import decode_time


LOG = logging.getLogger(__name__)


def clean_uri(uri):
    '''Cleans URI
    '''
    if uri.count('/') > 1:
        prefix = uri.split('/', 1)[0]
        if prefix.isalnum():
            uri = prefix.replace('.', '_')
        else:
            uri = ''
    if not uri:
        uri = '_other'

    return uri


def parse_log_lines(lines, pattern_fn):
    '''Parses log line using pattern
    '''
    for idx, line in enumerate(lines):
        match = pattern_fn(line)
        if not match:
            continue

        data = match.groupdict()
        if 'time' in data:
            data['time'] = decode_time(data['time'])
        if 'method' in data:
            data['method.' + data.pop('method')] = 1
        if 'uri' in data:
            data['uri'] = clean_uri(data['uri'])
        if 'http' in data:
            data['http.' + data.pop('http').replace('.', '_')] = 1

        for field in ['bbytes', 'creqs']:
            if field not in data:
                continue
            data[field] = int(data[field])

        for field in ['uctim', 'uhtim', 'urtim', 'gzip']:
            if field not in data:
                continue
            if data[field] == '-':
                del data[field]
            else:
                data[field] = float(data[field])

        if 'pipe' in data:
            if data['pipe'] != 'p':
                del data['pipe']

        LOG.debug('DATA: %s', repr(data))

        time = data.pop('time')
        if time:
            yield (idx, (time, data))
{'fraction_non_alphanumeric': 0.14534198113207547, 'fraction_numerical': 0.003242924528301887, 'mean_word_length': 1.734085414987913, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3208131', 'n_tokens_mistral': 1048, 'n_tokens_neox': 1006, 'n_words': 280}