<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
idcview.py

Created by QingFeng on 2008-03-17.
Copyright (c) 2007 xBayDNS Team. All rights reserved.
"""

from decimal import Decimal
from operator import itemgetter
import os, sys, time

from xbaydns.conf import sysconf

def convfiles(files):
    agents = []
    data = {}
    for filename in files:
        agent_name = os.path.basename(filename).split('_')[0]
        agents.append(agent_name)
        file_obj = open(filename, "r")
        line = file_obj.readline()
        try:
            preip, pingtype, latency, datetime = line.split(',')
        except Exception, e:
            print line
        if preip not in data:
            data[preip] = {}
        latency_sum = Decimal(latency)
        record_count = 1
        for line in file_obj:
            ip, pingtype, latency, datetime = line.split(',')
            if preip != ip:
                if preip not in data:
                    data[preip] = {}
                data[preip][agent_name] = latency_sum/record_count
                preip = ip
                latency_sum = Decimal(latency)
                record_count = 1
            else:
                latency_sum += Decimal(latency)
                record_count += 1
        if preip not in data:
            data[preip] = {}
        data[preip][agent_name] = latency_sum/record_count
    return (agents, data)

def main():
    if len(sys.argv) == 1:
        logdir = os.path.join(sysconf.xbaydnsdb, 'iplatency')
        if os.path.isdir(logdir) == False:
            print "No such a directory %s"%logdir
            sys.exit(1)
        else:
            agentfiles = os.listdir(logdir)
            agentfiles = map(lambda x:os.path.join(logdir, x), agentfiles)
    else:<commit_msg>drop damaged data<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
idcview.py

Created by QingFeng on 2008-03-17.
Copyright (c) 2007 xBayDNS Team. All rights reserved.
"""

from decimal import Decimal
from operator import itemgetter
import os, sys, time

from xbaydns.conf import sysconf

def convfiles(files):
    agents = []
    data = {}
    for filename in files:
        agent_name = os.path.basename(filename).split('_')[0]
        agents.append(agent_name)
        file_obj = open(filename, "r")
        line = file_obj.readline()
        try:
            preip, pingtype, latency, datetime = line.split(',')
        except Exception, e:
            print line
        if preip not in data:
            data[preip] = {}
        latency_sum = Decimal(latency)
        record_count = 1
        for line in file_obj:
            try:
                ip, pingtype, latency, datetime = line.split(',')
            except:
                continue
            if preip != ip:
                if preip not in data:
                    data[preip] = {}
                data[preip][agent_name] = latency_sum/record_count
                preip = ip
                latency_sum = Decimal(latency)
                record_count = 1
            else:
                latency_sum += Decimal(latency)
                record_count += 1
        if preip not in data:
            data[preip] = {}
        data[preip][agent_name] = latency_sum/record_count
    return (agents, data)

def main():
    if len(sys.argv) == 1:
        logdir = os.path.join(sysconf.xbaydnsdb, 'iplatency')
        if os.path.isdir(logdir) == False:
            print "No such a directory %s"%logdir
            sys.exit(1)
        else:
            agentfiles = os.listdir(logdir)
            agentfiles = map(lambda x:os.path.join(logdir, x), agentfiles)
    else:
{'fraction_non_alphanumeric': 0.1041551246537396, 'fraction_numerical': 0.010526315789473684, 'mean_word_length': 1.8166926677067083, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7566289', 'n_tokens_mistral': 1160, 'n_tokens_neox': 1093, 'n_words': 284}