<commit_before>import datetime
import json
import os
import re
import redis
import sys
import logging

cfg_path = os.path.realpath('config')
sys.path.append(cfg_path)

import gunicorn_config

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
        level=logging.INFO)
logger = logging.getLogger('metrics')

redis_opts = {}
redis_conn = None
cache_prefix = 'bandwidth_log:'
logging_period = 60 * 24  # 24hs
logging_interval = 15  # 15 minutes
exp_time = 60 * 60 * 24  # Key expires in 24hs
try:
    with open('/home/dotcloud/environment.json') as f:
        env = json.load(f)
        # Prod
        redis_opts = {
            'host': env['DOTCLOUD_REDIS_REDIS_HOST'],
            'port': int(env['DOTCLOUD_REDIS_REDIS_PORT']),
            'db': 1,
            'password': env['DOTCLOUD_REDIS_REDIS_PASSWORD'],
        }
except Exception:
    # Dev
    redis_opts = {
        'host': 'localhost',
        'port': 6380,
        'db': 0,
        'password': <PASSWORD>,
    }


def convert_str_to_datetime(date_str):
    return datetime.datetime.strptime(date_str, '%d/%b/%Y:%H:%M:%S')


def raw_line_parser(str_line):
    pattern = ("(?P<ip>\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) - - \["
               "(?P<date>\d{2}/\w+/\d{4}:\d{2}:\d{2}:\d{2})?\] \""
               "(?P<http_request>\w+)? /\w+/\w+/"
               "(?P<id>\w+)?/(?P<type>\w+)?")
    pattern_2 = ".*?(\d+)$"
    results = re.match(pattern, str_line)
    if results is None:
        return results
    results = re.match(pattern, str_line).groupdict()
    temp_results = re.match(pattern_2, str_line)
    if temp_results is None:
        results['size'] = None
        return results
    results['size'] = re.match(pattern_2, str_line).group(1)
    return results


def compute_bandwidth(str_end_time, str_start_time, str_layer_size):
    bandwidth = 0.0
    if str_start_time is None:
        return bandwidth
    if str_end_time is None:
        return bandwidth
    if str_layer_size is None:
        return bandwidth
    start_time = convert_str_to_datetime(str_start_time)
    end_time = convert_str_to_datetime(str_end_time)
    layer_size = long(str_layer_size)
    layer_size_mb = (layer_size * 8) / (1000 * 1000)  # Megabits
    delta = end_time - start_time
    num_seconds = delta.total_seconds()
    bandwidth = 0.0
    if num_seconds:
        bandwidth = layer_size_mb / num_seconds  # Megabits-per-second (Mbps)
    return bandwidth


def set_cache(key, bandwidth):
    global redis_conn, cache_prefix, exp_period
    if redis_conn is None:
        logger.info('Failed to find a redis connection.')
        return
    redis_conn.setex(key, exp_time, bandwidth)  # time in seconds
    logger.info('Saved in Redis: key: {0} bandwidth: {1}'.format(key, bandwidth))


def adjust_current_interval(current_interval, end_time, items):
    global logging_interval, logging_period
    total_items = logging_period / logging_interval
    logger.info('Skipping interval: {0}'.format(current_interval))
    for i in range(items, total_items):
        items = i + 1
        current_interval -= datetime.timedelta(minutes=logging_interval)
        if current_interval <= end_time:<commit_msg>Fixed pep8 warnings<commit_after>import datetime
import json
import logging
import os
import re
import redis
import sys

cfg_path = os.path.realpath('config')
sys.path.append(cfg_path)

import gunicorn_config

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                    level=logging.INFO)
logger = logging.getLogger('metrics')

redis_opts = {}
redis_conn = None
cache_prefix = 'bandwidth_log:'
logging_period = 60 * 24  # 24hs
logging_interval = 15  # 15 minutes
exp_time = 60 * 60 * 24  # Key expires in 24hs
try:
    with open('/home/dotcloud/environment.json') as f:
        env = json.load(f)
        # Prod
        redis_opts = {
            'host': env['DOTCLOUD_REDIS_REDIS_HOST'],
            'port': int(env['DOTCLOUD_REDIS_REDIS_PORT']),
            'db': 1,
            'password': env['DOTCLOUD_REDIS_REDIS_PASSWORD'],
        }
except Exception:
    # Dev
    redis_opts = {
        'host': 'localhost',
        'port': 6380,
        'db': 0,
        'password': <PASSWORD>,
    }


def convert_str_to_datetime(date_str):
    return datetime.datetime.strptime(date_str, '%d/%b/%Y:%H:%M:%S')


def raw_line_parser(str_line):
    pattern = ("(?P<ip>\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3}) - - \["
               "(?P<date>\d{2}/\w+/\d{4}:\d{2}:\d{2}:\d{2})?\] \""
               "(?P<http_request>\w+)? /\w+/\w+/"
               "(?P<id>\w+)?/(?P<type>\w+)?")
    pattern_2 = ".*?(\d+)$"
    results = re.match(pattern, str_line)
    if results is None:
        return results
    results = re.match(pattern, str_line).groupdict()
    temp_results = re.match(pattern_2, str_line)
    if temp_results is None:
        results['size'] = None
        return results
    results['size'] = re.match(pattern_2, str_line).group(1)
    return results


def compute_bandwidth(str_end_time, str_start_time, str_layer_size):
    bandwidth = 0.0
    if str_start_time is None:
        return bandwidth
    if str_end_time is None:
        return bandwidth
    if str_layer_size is None:
        return bandwidth
    start_time = convert_str_to_datetime(str_start_time)
    end_time = convert_str_to_datetime(str_end_time)
    layer_size = long(str_layer_size)
    layer_size_mb = (layer_size * 8) / (1000 * 1000)  # Megabits
    delta = end_time - start_time
    num_seconds = delta.total_seconds()
    bandwidth = 0.0
    if num_seconds:
        bandwidth = layer_size_mb / num_seconds  # Megabits-per-second (Mbps)
    return bandwidth


def set_cache(key, bandwidth):
    global redis_conn, cache_prefix, exp_period
    if redis_conn is None:
        logger.info('Failed to find a redis connection.')
        return
    redis_conn.setex(key, exp_time, bandwidth)  # time in seconds
    logger.info('Saved in Redis: key: {0} bandwidth: {1}'.format(key,
                                                                 bandwidth))


def adjust_current_interval(current_interval, end_time, items):
    global logging_interval, logging_period
    total_items = logging_period / logging_interval
    logger.info('Skipping interval: {0}'.format(current_interval))
    for i in range(items, total_items):
        items = i + 1
        current_interval -= datetime.timedelta(minutes=logging_interval)
        if current_interval <= end_time:
{'fraction_non_alphanumeric': 0.1296440230063734, 'fraction_numerical': 0.018187470853412095, 'mean_word_length': 2.9375764993880047, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6858581', 'n_tokens_mistral': 2281, 'n_tokens_neox': 2175, 'n_words': 457}