<commit_before>#!/usr/bin/env python

# Copyright (C) 2011 by <NAME> (<EMAIL>)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

""" Reports the state of your given job tree.
"""
import cPickle
import os
from random import choice
import string
import sys
import time

import xml.etree.ElementTree as ET  # not cElementTree so as to allow caching
from xml.dom import minidom  # For making stuff pretty

from sonLib.bioio import logger
from sonLib.bioio import logFile

from sonLib.bioio import getBasicOptionParser
from sonLib.bioio import parseBasicOptions
from sonLib.bioio import TempFileTree

from jobTree.src.master import getEnvironmentFileName, getJobFileDirName
from jobTree.src.master import getStatsFileName, getConfigFileName
from jobTree.src.master import getStatsCacheFileName

class JTTag(object):
    """ Convenience object that stores xml attributes as object attributes.
    """
    def __init__(self, tree):
        """ Given an ElementTree tag, build a convenience object.
        """
        for name in ["total_time", "median_clock", "total_memory",
                     "median_wait", "total_number", "average_time",
                     "median_memory", "min_number_per_slave", "average_wait",
                     "total_clock", "median_time", "min_time", "min_wait",
                     "max_clock", "max_wait", "total_wait", "min_clock",
                     "average_memory", "max_number_per_slave", "max_memory",
                     "average_memory", "max_number_per_slave", "max_memory",
                     "median_number_per_slave", "average_number_per_slave",
                     "max_time", "average_clock", "min_memory", "min_clock",
                     ]:
          setattr(self, name, self.__get(tree, name))
        self.name = tree.tag
    def __get(self, tag, name):
      if name in tag.attrib:
          value = tag.attrib[name]
      else:
          return float("nan")
      try:
          a = float(value)
      except ValueError:
          a = float("nan")
      return a

class ColumnWidths(object):
    """ Convenience object that stores the width of columns for printing.
    Helps make things pretty.
    """
    def __init__(self):
        self.count = 10
        self.min = 10
        self.med = 10
        self.ave = 10
        self.max = 10
        self.total = 10
    def title(self):
        """ Return the total printed length of this category item.
        """
        return self.min + self.med + self.ave + self.max + self.total

def initializeOptions(parser):
    ##########################################
    # Construct the arguments.
    ##########################################
    parser.add_option("--jobTree", dest="jobTree",
                      help="Directory containing the job tree")
    parser.add_option("--outputFile", dest="outputFile", default=None,
                      help="File in which to write results")
    parser.add_option("--raw", action="store_true", default=False,
                      help="output the raw xml data.")
    parser.add_option("--pretty", "--human", action="store_true", default=False,
                      help=("if not raw, prettify the numbers to be "
                            "human readable."))
    parser.add_option("--categories",
                      help=("comma separated list from [time, clock, wait, "
                            "memory]"))
    parser.add_option("--sortCategory", default="time",
                      help=("how to sort Target list. may be from [alpha, "
                            "time, clock, wait, memory, count]. "
                            "default=%(default)s"))
    parser.add_option("--sortField", default="med",
                      help=("how to sort Target list. may be from [min, "
                            "med, ave, max, total]. "
                            "default=%(default)s"))
    parser.add_option("--sortReverse", "--reverseSort", default=False,
                      action="store_true",
                      help="reverse sort order.")
    parser.add_option("--cache", default=False, action="store_true",
                      help="stores a cache to speed up data display.")

def checkOptions(options, args, parser):
    """ Check options, throw parser.error() if something goes wrong
    """
    logger.info("Parsed arguments")
    assert len(args) <= 1  # Only jobtree may be specified as argument
    if len(args) == 1:  # Allow jobTree directory as arg
        options.jobTree = args[0]
    logger.info("Checking if we have files for job tree")
    if options.jobTree == None:
        parser.error("Specify --jobTree")
    if not os.path.exists(options.jobTree):
        parser.error("--jobTree %s does not exist"
                     % options.jobTree)
    if not os.path.isdir(options.jobTree):
        parser.error("--jobTree %s is not a directory"
                     % options.jobTree)
    if not os.path.isfile(getConfigFileName(options.jobTree)):
        parser.error("A valid job tree must contain the config file")
    if not os.path.isfile(getStatsFileName(options.jobTree)):
        parser.error("The job-tree was run without the --stats flag, "
                     "so no stats were created")
    defaultCategories = ["time", "clock", "wait", "memory"]
    if options.categories is None:
        options.categories = defaultCategories
    else:
        options.categories = options.categories.split(",")
    for c in options.categories:
        if c not in defaultCategories:
            parser.error("Unknown category %s. Must be from %s"
                         % (c, str(defaultCategories)))
    extraSort = ["count", "alpha"]
    if options.sortCategory is not None:
        if (options.sortCategory not in defaultCategories and
            options.sortCategory not in extraSort):
            parser.error("Unknown --sortCategory %s. Must be from %s"
                         % (options.sortCategory,
                            str(defaultCategories + extraSort)))
    sortFields = ["min", "med", "ave", "max", "total"]
    if options.sortField is not None:
        if (options.sortField not in sortFields):
            parser.error("Unknown --sortField %s. Must be from %s"
                         % (options.sortField, str(sortFields)))
    logger.info("Checked arguments")

def prettyXml(elem):
    """ Return a pretty-printed XML string for the ElementTree Element.
    """
    roughString = ET.tostring(elem, "utf-8")
    reparsed = minidom.parseString(roughString)
    return reparsed.toprettyxml(indent="  ")<commit_msg>allowing title casing on category names<commit_after>#!/usr/bin/env python

# Copyright (C) 2011 by <NAME> (<EMAIL>)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

""" Reports the state of your given job tree.
"""
import cPickle
import os
from random import choice
import string
import sys
import time

import xml.etree.ElementTree as ET  # not cElementTree so as to allow caching
from xml.dom import minidom  # For making stuff pretty

from sonLib.bioio import logger
from sonLib.bioio import logFile

from sonLib.bioio import getBasicOptionParser
from sonLib.bioio import parseBasicOptions
from sonLib.bioio import TempFileTree

from jobTree.src.master import getEnvironmentFileName, getJobFileDirName
from jobTree.src.master import getStatsFileName, getConfigFileName
from jobTree.src.master import getStatsCacheFileName

class JTTag(object):
    """ Convenience object that stores xml attributes as object attributes.
    """
    def __init__(self, tree):
        """ Given an ElementTree tag, build a convenience object.
        """
        for name in ["total_time", "median_clock", "total_memory",
                     "median_wait", "total_number", "average_time",
                     "median_memory", "min_number_per_slave", "average_wait",
                     "total_clock", "median_time", "min_time", "min_wait",
                     "max_clock", "max_wait", "total_wait", "min_clock",
                     "average_memory", "max_number_per_slave", "max_memory",
                     "average_memory", "max_number_per_slave", "max_memory",
                     "median_number_per_slave", "average_number_per_slave",
                     "max_time", "average_clock", "min_memory", "min_clock",
                     ]:
          setattr(self, name, self.__get(tree, name))
        self.name = tree.tag
    def __get(self, tag, name):
      if name in tag.attrib:
          value = tag.attrib[name]
      else:
          return float("nan")
      try:
          a = float(value)
      except ValueError:
          a = float("nan")
      return a

class ColumnWidths(object):
    """ Convenience object that stores the width of columns for printing.
    Helps make things pretty.
    """
    def __init__(self):
        self.count = 10
        self.min = 10
        self.med = 10
        self.ave = 10
        self.max = 10
        self.total = 10
    def title(self):
        """ Return the total printed length of this category item.
        """
        return self.min + self.med + self.ave + self.max + self.total

def initializeOptions(parser):
    ##########################################
    # Construct the arguments.
    ##########################################
    parser.add_option("--jobTree", dest="jobTree",
                      help="Directory containing the job tree")
    parser.add_option("--outputFile", dest="outputFile", default=None,
                      help="File in which to write results")
    parser.add_option("--raw", action="store_true", default=False,
                      help="output the raw xml data.")
    parser.add_option("--pretty", "--human", action="store_true", default=False,
                      help=("if not raw, prettify the numbers to be "
                            "human readable."))
    parser.add_option("--categories",
                      help=("comma separated list from [time, clock, wait, "
                            "memory]"))
    parser.add_option("--sortCategory", default="time",
                      help=("how to sort Target list. may be from [alpha, "
                            "time, clock, wait, memory, count]. "
                            "default=%(default)s"))
    parser.add_option("--sortField", default="med",
                      help=("how to sort Target list. may be from [min, "
                            "med, ave, max, total]. "
                            "default=%(default)s"))
    parser.add_option("--sortReverse", "--reverseSort", default=False,
                      action="store_true",
                      help="reverse sort order.")
    parser.add_option("--cache", default=False, action="store_true",
                      help="stores a cache to speed up data display.")

def checkOptions(options, args, parser):
    """ Check options, throw parser.error() if something goes wrong
    """
    logger.info("Parsed arguments")
    assert len(args) <= 1  # Only jobtree may be specified as argument
    if len(args) == 1:  # Allow jobTree directory as arg
        options.jobTree = args[0]
    logger.info("Checking if we have files for job tree")
    if options.jobTree == None:
        parser.error("Specify --jobTree")
    if not os.path.exists(options.jobTree):
        parser.error("--jobTree %s does not exist"
                     % options.jobTree)
    if not os.path.isdir(options.jobTree):
        parser.error("--jobTree %s is not a directory"
                     % options.jobTree)
    if not os.path.isfile(getConfigFileName(options.jobTree)):
        parser.error("A valid job tree must contain the config file")
    if not os.path.isfile(getStatsFileName(options.jobTree)):
        parser.error("The job-tree was run without the --stats flag, "
                     "so no stats were created")
    defaultCategories = ["time", "clock", "wait", "memory"]
    if options.categories is None:
        options.categories = defaultCategories
    else:
        options.categories = map(lambda x: x.lower(),
                                 options.categories.split(","))
    for c in options.categories:
        if c not in defaultCategories:
            parser.error("Unknown category %s. Must be from %s"
                         % (c, str(defaultCategories)))
    extraSort = ["count", "alpha"]
    if options.sortCategory is not None:
        if (options.sortCategory not in defaultCategories and
            options.sortCategory not in extraSort):
            parser.error("Unknown --sortCategory %s. Must be from %s"
                         % (options.sortCategory,
                            str(defaultCategories + extraSort)))
    sortFields = ["min", "med", "ave", "max", "total"]
    if options.sortField is not None:
        if (options.sortField not in sortFields):
            parser.error("Unknown --sortField %s. Must be from %s"
                         % (options.sortField, str(sortFields)))
    logger.info("Checked arguments")

def prettyXml(elem):
    """ Return a pretty-printed XML string for the ElementTree Element.
    """
    roughString = ET.tostring(elem, "utf-8")
    reparsed = minidom.parseString(roughString)
    return reparsed.toprettyxml(indent="  ")
{'fraction_non_alphanumeric': 0.11238547228264452, 'fraction_numerical': 0.002636609320413948, 'mean_word_length': 2.471059254175246, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2839213', 'n_tokens_mistral': 4084, 'n_tokens_neox': 3906, 'n_words': 1360}