<commit_before>#!/usr/bin/python

import sys, re, pdb
import logging
import argparse

import numpy as np, pandas as pd

import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')

import utils, data_helper, analysis

def parse_args():
    parser = argparse.ArgumentParser(description='SOPERNOVUS analysator')
    parser.add_argument('-i','--input-file', help='Input CSV file', required=True)
    parser.add_argument('-o','--output-file', help='Output CSV file', required=False, default = None)
    args = vars(parser.parse_args())
    return args

def main():
    args = parse_args()
    inputFileName = args['input_file']
    outputFileName = args['output_file']

    utils.log_config()
    logger = logging.getLogger(__name__)

    logger.info("Data file: {}".format(inputFileName))

    # exported to global scope for debugging purposes
    global df

    df = data_helper.import_data(inputFileName)

    logger.info("N of events: {}, from {} to {} ".format(len(df), df['datetime'].min(), df['datetime'].max()))

    # exported to global scope for debugging purposes
    global apps
    apps =  analysis.summarize_applications(df)

    logger.info("N of applications = {}".format(len(apps)))

    print(apps)

    if outputFileName is not None:<commit_msg>command line args<commit_after>#!/usr/bin/python

import sys, re, pdb
import logging
import argparse

import numpy as np, pandas as pd

import matplotlib.pyplot as plt
import matplotlib
matplotlib.style.use('ggplot')

import utils, data_helper, analysis

def parse_args():
    parser = argparse.ArgumentParser(description='SOPERNOVUS analysator')
    parser.add_argument('--input-file', help='Input CSV file', required=True)
    parser.add_argument('--output-file', help='Output CSV file', required=False, default = None)
    args = vars(parser.parse_args())
    return args

def main():
    args = parse_args()
    inputFileName = args['input_file']
    outputFileName = args['output_file']

    utils.log_config()
    logger = logging.getLogger(__name__)

    logger.info("Data file: {}".format(inputFileName))

    # exported to global scope for debugging purposes
    global df

    df = data_helper.import_data(inputFileName)

    logger.info("N of events: {}, from {} to {} ".format(len(df), df['datetime'].min(), df['datetime'].max()))

    # exported to global scope for debugging purposes
    global apps
    apps =  analysis.summarize_applications(df)

    logger.info("N of applications = {}".format(len(apps)))

    print(apps)

    if outputFileName is not None:
{'fraction_non_alphanumeric': 0.12832550860719874, 'fraction_numerical': 0.0, 'mean_word_length': 4.186612576064909, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4684872', 'n_tokens_mistral': 779, 'n_tokens_neox': 748, 'n_words': 205}