<commit_before>#!/usr/bin/python
# encoding: utf-8
"""

@author: xuk1
@license: (C) Copyright 2013-2017
@contact: <EMAIL>
@file: processing.py
@time: 8/23/2017 13:57
@desc: 

"""
import argparse
import datetime
import os
import sys
import time

import pandas as pd

from bb_parse import BBParse
from cluster import Cluster
from parallelprocessing import get_cluster_data_by_time


def env_check():
    """
    Checking the running environment
    :return: 
    """
    py_version = sys.version_info
    if py_version[:2] >= (2, 7):
        print "---- You currently have Python " + sys.version
    else:
        print "---- Error, You need python 2.7.x+ and currently you have " + sys.version + 'exiting now...'
        exit(-1)
    try:
        import numpy, pandas
    except ImportError:
        print '---- missing dependency: numpy or pandas, please install first'
        exit(-1)

    try:
        import tables  # noqa
    except ImportError as ex:  # pragma: no cover
        raise ImportError('HDFStore requires PyTables, "{ex}" problem '
                          'importing'.format(ex=str(ex)))

    print '---- You have all required dependencies, starting to process'


def save_avg_result(*option):
    """
    Save results to file
    :param option: optional inputs can be: save_avg_result(pat_path) or save_avg_result(pat_path, bb_log_path) or 
    save_avg_result(pat_path, bb_log_path, BB_Phase) 
    :return: None
    """
    if len(option) == 1:  # only pat_path is assigned
        result_file = option[0] + os.sep + 'results.txt'
        attrib_avg = Cluster(option[0]).get_cluster_data_by_time([0], [0], False)
        with open(result_file, 'w') as f:
            f.write('*' * 110 + '\n')
            f.write('All nodes average utilization\n')
            f.write('*' * 110 + '\n')
            for key in attrib_avg.keys():
                f.write('All nodes average {0} utilization: \n {1} \n'
                        .format(key, attrib_avg.get(key).to_string(index=False)))
                f.write('.' * 75 + '\n')
        print 'Results have been saved to: {0}'.format(result_file)
        return
    elif len(option) == 2:  # pat_path and bb_log are assigned
        result_file = option[0] + os.sep + 'results.txt'
        phase_name = ('BENCHMARK', 'LOAD_TEST', 'POWER_TEST', 'THROUGHPUT_TEST_1',
                      'VALIDATE_POWER_TEST', 'VALIDATE_THROUGHPUT_TEST_1')
        with open(result_file, 'w') as f:
            for phase in phase_name[0:4]:
                start_stamp, end_stamp = BBParse(option[1]).get_stamp_by_phase(phase)
                start_time = datetime.fromtimestamp(start_stamp).strftime('%Y-%m-%d %H:%M:%S')
                end_time = datetime.fromtimestamp(end_stamp).strftime('%Y-%m-%d %H:%M:%S')
                attrib_avg = Cluster(option[0]).get_cluster_avg(start_stamp, end_stamp)
                f.write('*' * 110 + '\n')
                f.write('All nodes average utilization for phase {0} between {1} and {2}:\n'
                        .format(phase, start_time, end_time))
                f.write('*' * 110 + '\n')
                for key in attrib_avg.keys():
                    f.write('All nodes average {0} utilization: \n {1} \n'
                            .format(key, attrib_avg.get(key).to_string(index=False)))
                    f.write('.' * 75 + '\n')
        print 'Results have been saved to: {0}'.format(result_file)
        return
    elif len(option) == 3:  # pat_path, bb_log and phase_name are assigned
        result_file = option[0] + os.sep + 'results.txt'
        with open(result_file, 'w') as f:
            start_stamp, end_stamp = BBParse(option[1]).get_stamp_by_phase(option[2])
            start_time = datetime.fromtimestamp(start_stamp).strftime('%Y-%m-%d %H:%M:%S')
            end_time = datetime.fromtimestamp(end_stamp).strftime('%Y-%m-%d %H:%M:%S')
            attrib_avg = Cluster(option[0]).get_cluster_avg(start_stamp, end_stamp)
            f.write('*' * 110 + '\n')
            f.write('All nodes average utilization for phase {0} between {1} and {2}:\n'
                    .format(option[2], start_time, end_time))
            f.write('*' * 110 + '\n')
            for key in attrib_avg.keys():
                f.write('All nodes average {0} utilization: \n {1} \n'
                        .format(key, attrib_avg.get(key).to_string(index=False)))
                f.write('.' * 75 + '\n')
        print 'Results have been saved to: {0}'.format(result_file)
        return
    else:
        print 'Usage: save_avg_result(pat_path) or save_avg_result(pat_path, bb_log_path) or ' \
              'save_avg_result(pat_path, bb_log_path, BB_Phase)\n'
        exit(-1)


def str2bool(v):
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')


def get_args():
    """
    Parse command line inputs
    :return: 
    """
    parse = argparse.ArgumentParser(description='Processing PAT data')
    group = parse.add_mutually_exclusive_group()

    parse.add_argument('-p', '--pat', type=str, help='PAT file path', required=True)
    parse.add_argument('-l', '--log', type=str, help='TPCx-BB log path', required=False)
    group.add_argument('-ph', '--phase', type=str, help='TPCx-BB phase', required=False, nargs='+', default='BENCHMARK')
    group.add_argument('-q', '--query', type=int, help='TPCx-BB query num', required=False, nargs='+')
    group.add_argument('-n', '--streamNumber', type=int, help='TPCx-BB stream number', required=False, nargs='+')
    parse.add_argument('-s', '--save', type=str2bool, help='whether to save raw data', required=False, default=False)

    args = parse.parse_args()
    pat_path = args.pat
    log_path = args.log
    phase = args.phase
    stream = args.streamNumber
    query = args.query
    save_raw = args.save
    return pat_path, log_path, phase, stream, query, save_raw


def run():
    env_check()
    pat_path, log_path, phase, stream, query, save_raw = get_args()
    if os.path.exists(pat_path):
        if not log_path:  # only pat_path is assigned
            print 'only pat_path is assigned, calculating BENCHMARK average utilization...\n'
            cluster_avg = get_cluster_data_by_time(pat_path, [0], [0], save_raw)
            tag = [0]
            print_pat_result(cluster_avg, tag)
            result_path = pat_path + os.sep + 'pat_avg.txt'
            save_pat_result(cluster_avg, tag, result_path)
        else:  # pat_path and log_path are assigned
            if os.path.exists(log_path):
                phase_ts = BBParse(log_path).get_exist_phase_timestamp()
            else:
                print 'TPCx-BB log file path: {0} does not exist, exiting...'.format(log_path)
                exit(-1)

            start_stamps = []
            end_stamps = []
            if (not query) & (not stream) & (phase == 'BENCHMARK'):  # if -ph -q and -n not assigned
                for key, value in phase_ts.items():
                    start_stamps.extend((value['epochStartTimestamp'] / 1000).tolist())
                    end_stamps.extend((value['epochEndTimestamp'] / 1000).tolist())
                assert len(start_stamps) == len(end_stamps)
                # cluster_avg = Cluster(pat_path).get_cluster_data_by_time(start_stamps, end_stamps, save_raw)
                cluster_avg = get_cluster_data_by_time(pat_path, start_stamps, end_stamps, save_raw)
                bb_result = pd.concat(phase_ts.values(), axis=0).reset_index(drop=True)
                pat_result = pd.concat(cluster_avg.values(), axis=1)
                avg_result = pd.concat([bb_result, pat_result], axis=1)
                result_path = pat_path + os.sep + 'results.txt'
                avg_result.to_csv(result_path, sep=',')
                tag = []
                for key in phase_ts.keys():<commit_msg>add parse BB log to default<commit_after>#!/usr/bin/python
# encoding: utf-8
"""

@author: xuk1
@license: (C) Copyright 2013-2017
@contact: <EMAIL>
@file: processing.py
@time: 8/23/2017 13:57
@desc: 

"""
import argparse
import datetime
import os
import sys
import time

import pandas as pd

from bb_parse import BBParse
from cluster import Cluster
from parallelprocessing import get_cluster_data_by_time


def env_check():
    """
    Checking the running environment
    :return: 
    """
    py_version = sys.version_info
    if py_version[:2] >= (2, 7):
        print "---- You currently have Python " + sys.version
    else:
        print "---- Error, You need python 2.7.x+ and currently you have " + sys.version + 'exiting now...'
        exit(-1)
    try:
        import numpy, pandas
    except ImportError:
        print '---- missing dependency: numpy or pandas, please install first'
        exit(-1)

    try:
        import tables  # noqa
    except ImportError as ex:  # pragma: no cover
        raise ImportError('HDFStore requires PyTables, "{ex}" problem '
                          'importing'.format(ex=str(ex)))

    print '---- You have all required dependencies, starting to process'


def save_avg_result(*option):
    """
    Save results to file
    :param option: optional inputs can be: save_avg_result(pat_path) or save_avg_result(pat_path, bb_log_path) or 
    save_avg_result(pat_path, bb_log_path, BB_Phase) 
    :return: None
    """
    if len(option) == 1:  # only pat_path is assigned
        result_file = option[0] + os.sep + 'results.txt'
        attrib_avg = Cluster(option[0]).get_cluster_data_by_time([0], [0], False)
        with open(result_file, 'w') as f:
            f.write('*' * 110 + '\n')
            f.write('All nodes average utilization\n')
            f.write('*' * 110 + '\n')
            for key in attrib_avg.keys():
                f.write('All nodes average {0} utilization: \n {1} \n'
                        .format(key, attrib_avg.get(key).to_string(index=False)))
                f.write('.' * 75 + '\n')
        print 'Results have been saved to: {0}'.format(result_file)
        return
    elif len(option) == 2:  # pat_path and bb_log are assigned
        result_file = option[0] + os.sep + 'results.txt'
        phase_name = ('BENCHMARK', 'LOAD_TEST', 'POWER_TEST', 'THROUGHPUT_TEST_1',
                      'VALIDATE_POWER_TEST', 'VALIDATE_THROUGHPUT_TEST_1')
        with open(result_file, 'w') as f:
            for phase in phase_name[0:4]:
                start_stamp, end_stamp = BBParse(option[1]).get_stamp_by_phase(phase)
                start_time = datetime.fromtimestamp(start_stamp).strftime('%Y-%m-%d %H:%M:%S')
                end_time = datetime.fromtimestamp(end_stamp).strftime('%Y-%m-%d %H:%M:%S')
                attrib_avg = Cluster(option[0]).get_cluster_avg(start_stamp, end_stamp)
                f.write('*' * 110 + '\n')
                f.write('All nodes average utilization for phase {0} between {1} and {2}:\n'
                        .format(phase, start_time, end_time))
                f.write('*' * 110 + '\n')
                for key in attrib_avg.keys():
                    f.write('All nodes average {0} utilization: \n {1} \n'
                            .format(key, attrib_avg.get(key).to_string(index=False)))
                    f.write('.' * 75 + '\n')
        print 'Results have been saved to: {0}'.format(result_file)
        return
    elif len(option) == 3:  # pat_path, bb_log and phase_name are assigned
        result_file = option[0] + os.sep + 'results.txt'
        with open(result_file, 'w') as f:
            start_stamp, end_stamp = BBParse(option[1]).get_stamp_by_phase(option[2])
            start_time = datetime.fromtimestamp(start_stamp).strftime('%Y-%m-%d %H:%M:%S')
            end_time = datetime.fromtimestamp(end_stamp).strftime('%Y-%m-%d %H:%M:%S')
            attrib_avg = Cluster(option[0]).get_cluster_avg(start_stamp, end_stamp)
            f.write('*' * 110 + '\n')
            f.write('All nodes average utilization for phase {0} between {1} and {2}:\n'
                    .format(option[2], start_time, end_time))
            f.write('*' * 110 + '\n')
            for key in attrib_avg.keys():
                f.write('All nodes average {0} utilization: \n {1} \n'
                        .format(key, attrib_avg.get(key).to_string(index=False)))
                f.write('.' * 75 + '\n')
        print 'Results have been saved to: {0}'.format(result_file)
        return
    else:
        print 'Usage: save_avg_result(pat_path) or save_avg_result(pat_path, bb_log_path) or ' \
              'save_avg_result(pat_path, bb_log_path, BB_Phase)\n'
        exit(-1)


def str2bool(v):
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')


def get_args():
    """
    Parse command line inputs
    :return: 
    """
    parse = argparse.ArgumentParser(description='Processing PAT data')
    group = parse.add_mutually_exclusive_group()

    parse.add_argument('-p', '--pat', type=str, help='PAT file path', required=True)
    parse.add_argument('-l', '--log', type=str, help='TPCx-BB log path', required=False)
    group.add_argument('-ph', '--phase', type=str, help='TPCx-BB phase', required=False, nargs='+', default='BENCHMARK')
    group.add_argument('-q', '--query', type=int, help='TPCx-BB query num', required=False, nargs='+')
    group.add_argument('-n', '--streamNumber', type=int, help='TPCx-BB stream number', required=False, nargs='+')
    parse.add_argument('-s', '--save', type=str2bool, help='whether to save raw data', required=False, default=False)

    args = parse.parse_args()
    pat_path = args.pat
    log_path = args.log
    phase = args.phase
    stream = args.streamNumber
    query = args.query
    save_raw = args.save
    return pat_path, log_path, phase, stream, query, save_raw


def run():
    env_check()
    pat_path, log_path, phase, stream, query, save_raw = get_args()
    if os.path.exists(pat_path):
        if not log_path:  # only pat_path is assigned
            print 'only pat_path is assigned, calculating BENCHMARK average utilization...\n'
            cluster_avg = get_cluster_data_by_time(pat_path, [0], [0], save_raw)
            tag = [0]
            print_pat_result(cluster_avg, tag)
            result_path = pat_path + os.sep + 'pat_avg.txt'
            save_pat_result(cluster_avg, tag, result_path)
        else:  # pat_path and log_path are assigned
            if os.path.exists(log_path):
                bb_parse = BBParse(log_path)
                print 'Parsing TPCx-BB log files...\n'
                bb_parse.get_elapsed_time()
                print 'Started to process PAT files...\n'
                phase_ts = bb_parse.get_exist_phase_timestamp()
            else:
                print 'TPCx-BB log file path: {0} does not exist, exiting...'.format(log_path)
                exit(-1)

            start_stamps = []
            end_stamps = []
            if (not query) & (not stream) & (phase == 'BENCHMARK'):  # if -ph -q and -n not assigned
                for key, value in phase_ts.items():
                    start_stamps.extend((value['epochStartTimestamp'] / 1000).tolist())
                    end_stamps.extend((value['epochEndTimestamp'] / 1000).tolist())
                assert len(start_stamps) == len(end_stamps)
                # cluster_avg = Cluster(pat_path).get_cluster_data_by_time(start_stamps, end_stamps, save_raw)
                cluster_avg = get_cluster_data_by_time(pat_path, start_stamps, end_stamps, save_raw)
                bb_result = pd.concat(phase_ts.values(), axis=0).reset_index(drop=True)
                pat_result = pd.concat(cluster_avg.values(), axis=1)
                avg_result = pd.concat([bb_result, pat_result], axis=1)
                result_path = pat_path + os.sep + 'results.txt'
                avg_result.to_csv(result_path, sep=',')
                tag = []
                for key in phase_ts.keys():
{'fraction_non_alphanumeric': 0.1360666957008648, 'fraction_numerical': 0.013314253717414298, 'mean_word_length': 2.383287728899179, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1944741', 'n_tokens_mistral': 5354, 'n_tokens_neox': 5071, 'n_words': 1222}