<commit_before>""" 
A basic CLI for examining the logs and their analysis.
"""

import config
import notify
from log import Log
from messageQueue import Queue
from summaryTable import SummaryTable

import json
import optparse
import pickle

def printSummary():
  log = Log()
  log.debug("Displaying summary for queue called '" + config.AZURE_STORAGE_QUEUE_NAME + "' and table called '" + config.AZURE_STORAGE_SUMMARY_TABLE_NAME + "' in storage account called '" + config.AZURE_STORAGE_ACCOUNT_NAME + "'")

  table_service = getTableService() 
  queue_service = getQueueService()
  summary = "Queue Length is approximately: " + queue_service.getLength() + "\n\n"
  summary = summary + "Processed events:\n"
  summary = summary + "Errors: " + str(table_service.getCount("ERROR")) + "\n"
  summary = summary + "Warnings: " + str(table_service.getCount("WARNING")) + "\n"
  summary = summary + "Infos: " + str(table_service.getCount("INFO")) + "\n"
  summary = summary + "Debugs: " + str(table_service.getCount("DEBUG")) + "\n"
  summary = summary + "Others: " + str(table_service.getCount("OTHER")) + "\n"
  print(summary)
  notify.info(summary)

def getQueueService():
  queue_service = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
  return queue_service

def getTableService():
  return SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)

def dumpUnprocessedLogs():
  print ("Unproccessed logs")
  try:
      with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
          log = f.read()
      print (log)
  except:
      print("No logs waiting to be processed")

def dumpLogs():<commit_msg>output the count of correct and incorrect<commit_after>""" 
A basic CLI for examining the logs and their analysis.
"""

import config
import notify
from log import Log
from messageQueue import Queue
from summaryTable import SummaryTable

import json
import optparse
import pickle

def printSummary():
  log = Log()
  log.debug("Displaying summary for queue called '" + config.AZURE_STORAGE_QUEUE_NAME + "' and table called '" + config.AZURE_STORAGE_SUMMARY_TABLE_NAME + "' in storage account called '" + config.AZURE_STORAGE_ACCOUNT_NAME + "'")

  table_service = getTableService() 
  queue_service = getQueueService()
  summary = "Queue Length is approximately: " + queue_service.getLength() + "\n\n"
  summary = summary + "Processed events:\n"
  summary = summary + "Errors: " + str(table_service.getCount("ERROR")) + "\n"
  summary = summary + "Warnings: " + str(table_service.getCount("WARNING")) + "\n"
  summary = summary + "Infos: " + str(table_service.getCount("INFO")) + "\n"
  summary = summary + "Debugs: " + str(table_service.getCount("DEBUG")) + "\n"
  summary = summary + "Correct: " + str(table_service.getCount("CORRECT")) + "\n"
  summary = summary + "Incorrect: " + str(table_service.getCount("INCORRECT")) + "\n"
  summary = summary + "Others: " + str(table_service.getCount("OTHER")) + "\n"
  print(summary)
  notify.info(summary)

def getQueueService():
  queue_service = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
  return queue_service

def getTableService():
  return SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)

def dumpUnprocessedLogs():
  print ("Unproccessed logs")
  try:
      with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
          log = f.read()
      print (log)
  except:
      print("No logs waiting to be processed")

def dumpLogs():
{'fraction_non_alphanumeric': 0.12224017232094776, 'fraction_numerical': 0.0, 'mean_word_length': 5.120263591433279, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4892090', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1194, 'n_words': 281}