<commit_before>#
#  Copyright (c) 2015, 2016, 2017, 2018, 2019, Intel Corporation
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions
#  are met:
#
#      * Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#
#      * Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in
#        the documentation and/or other materials provided with the
#        distribution.
#
#      * Neither the name of Intel Corporation nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY LOG OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""
GEOPM IO - Helper module for parsing/processing report and trace files.
"""

import os
import json
import re
import pandas
import numpy
import glob
import sys
import subprocess
import psutil
import copy
import tempfile
import yaml
import StringIO

from distutils.spawn import find_executable
from natsort import natsorted
from geopmpy import __version__
from geopmpy import agent

try:
    _, os.environ['COLUMNS'] = subprocess.check_output(['stty', 'size']).split()
except subprocess.CalledProcessError:
    os.environ['COLUMNS'] = "200"

pandas.set_option('display.width', int(os.environ['COLUMNS']))
pandas.set_option('display.max_colwidth', 80)
pandas.set_option('max_columns', 100)


class AppOutput(object):
    """The container class for all report and trace related data.

    This class holds the relevant objects for parsing and indexing all
    data that is output from GEOPM.  This object can be created with a
    report glob string, a trace glob string, or both that will be used
    to search dir_name for the relevant files.  If files are found
    their data will be parsed into objects for easy data access.
    Additionally a Pandas DataFrame is constructed containing all of
    the report data and a separate DataFrame containing all of the
    trace data.  These DataFrames are indexed based on the version of
    GEOPM found in the files, the profile name, agent name, and the number
    of times that particular configuration has been seen by the parser
    (i.e. experiment iteration).

    Attributes:
        report_glob: The string pattern to use to search for report files.
        trace_glob: The string pattern to use to search for trace files.
        dir_name: The directory path to use when searching for files.
        verbose: A bool to control whether verbose output is printed to stdout.

    """

    def __init__(self, reports=None, traces=None, dir_name='.', verbose=False, do_cache=True):
        self._reports = {}
        self._reports_df = pandas.DataFrame()
        self._traces = {}
        self._traces_df = pandas.DataFrame()
        self._all_paths = []
        self._index_tracker = IndexTracker()
        self._node_names = None
        self._region_names = None

        if reports:
            if type(reports) is str:
                report_glob = os.path.join(dir_name, reports)
                report_paths = natsorted(glob.glob(report_glob))
                if len(report_paths) == 0:
                    raise RuntimeError('No report files found with pattern {}.'.format(report_glob))
            elif type(reports) is list:
                report_paths = [os.path.join(dir_name, path) for path in reports]
            else:
                raise TypeError('AppOutput: reports must be a list of paths or a glob pattern')

            self._all_paths.extend(report_paths)

            if do_cache:
                # unique cache name based on report files in this list
                # if all reports share a directory, put cache there
                dirs = set()
                for rr in report_paths:
                    dirs.add(rr)
                dirs = list(dirs)
                h5_dir = dirs[0] if len(dirs) == 1 else '.'
                paths_str = str(report_paths)
                report_h5_name = os.path.join(dir_name, 'report_{}.h5'.format(hash(paths_str)))
                self._all_paths.append(report_h5_name)

                # check if cache is older than reports
                if os.path.exists(report_h5_name):
                    cache_mod_time = os.path.getmtime(report_h5_name)
                    regen_cache = False
                    for report_file in report_paths:
                        mod_time = os.path.getmtime(report_file)
                        if mod_time > cache_mod_time:
                            regen_cache = True
                    if regen_cache:
                        os.remove(report_h5_name)

                try:
                    # load dataframes from cache
                    self._reports_df = pandas.read_hdf(report_h5_name, 'report')
                    self._app_reports_df = pandas.read_hdf(report_h5_name, 'app_report')
                    if verbose:
                        sys.stdout.write('Loaded reports from {}.\n'.format(report_h5_name))
                except IOError as err:
                    sys.stderr.write('<geopmpy>: Warning: report HDF5 file not detected or older than reports.  Data will be saved to {}.\n'
                                     .format(report_h5_name))
                    self.parse_reports(report_paths, verbose)

                    # Cache report dataframe
                    try:
                        if verbose:
                            sys.stdout.write('Generating HDF5 files... ')
                            self._reports_df.to_hdf(report_h5_name, 'report', format='table')
                            self._app_reports_df.to_hdf(report_h5_name, 'app_report', format='table', append=True)
                    except ImportError as error:
                        sys.stderr.write('<geopmy> Warning: unable to write HDF5 file: {}\n'.format(str(error)))

                    if verbose:
                        sys.stdout.write('Done.\n')
                        sys.stdout.flush()
            else:
                self.parse_reports(report_paths, verbose)

        if traces:
            if type(traces) is str:
                trace_glob = os.path.join(dir_name, traces)
                trace_paths = natsorted(glob.glob(trace_glob))
                if len(trace_paths) == 0:
                    raise RuntimeError('No trace files found with pattern {}.'.format(trace_glob))
            elif type(traces) is list:<commit_msg>Fix generation of report hdf5 cache<commit_after>#
#  Copyright (c) 2015, 2016, 2017, 2018, 2019, Intel Corporation
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions
#  are met:
#
#      * Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#
#      * Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in
#        the documentation and/or other materials provided with the
#        distribution.
#
#      * Neither the name of Intel Corporation nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY LOG OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
"""
GEOPM IO - Helper module for parsing/processing report and trace files.
"""

import os
import json
import re
import pandas
import numpy
import glob
import sys
import subprocess
import psutil
import copy
import tempfile
import yaml
import StringIO

from distutils.spawn import find_executable
from natsort import natsorted
from geopmpy import __version__
from geopmpy import agent

try:
    _, os.environ['COLUMNS'] = subprocess.check_output(['stty', 'size']).split()
except subprocess.CalledProcessError:
    os.environ['COLUMNS'] = "200"

pandas.set_option('display.width', int(os.environ['COLUMNS']))
pandas.set_option('display.max_colwidth', 80)
pandas.set_option('max_columns', 100)


class AppOutput(object):
    """The container class for all report and trace related data.

    This class holds the relevant objects for parsing and indexing all
    data that is output from GEOPM.  This object can be created with a
    report glob string, a trace glob string, or both that will be used
    to search dir_name for the relevant files.  If files are found
    their data will be parsed into objects for easy data access.
    Additionally a Pandas DataFrame is constructed containing all of
    the report data and a separate DataFrame containing all of the
    trace data.  These DataFrames are indexed based on the version of
    GEOPM found in the files, the profile name, agent name, and the number
    of times that particular configuration has been seen by the parser
    (i.e. experiment iteration).

    Attributes:
        report_glob: The string pattern to use to search for report files.
        trace_glob: The string pattern to use to search for trace files.
        dir_name: The directory path to use when searching for files.
        verbose: A bool to control whether verbose output is printed to stdout.

    """

    def __init__(self, reports=None, traces=None, dir_name='.', verbose=False, do_cache=True):
        self._reports = {}
        self._reports_df = pandas.DataFrame()
        self._traces = {}
        self._traces_df = pandas.DataFrame()
        self._all_paths = []
        self._index_tracker = IndexTracker()
        self._node_names = None
        self._region_names = None

        if reports:
            if type(reports) is str:
                report_glob = os.path.join(dir_name, reports)
                report_paths = natsorted(glob.glob(report_glob))
                if len(report_paths) == 0:
                    raise RuntimeError('No report files found with pattern {}.'.format(report_glob))
            elif type(reports) is list:
                report_paths = [os.path.join(dir_name, path) for path in reports]
            else:
                raise TypeError('AppOutput: reports must be a list of paths or a glob pattern')

            self._all_paths.extend(report_paths)

            if do_cache:
                # unique cache name based on report files in this list
                # if all reports share a directory, put cache there
                dirs = set()
                for rr in report_paths:
                    dirs.add(rr)
                dirs = list(dirs)
                h5_dir = dirs[0] if len(dirs) == 1 else '.'
                paths_str = str(report_paths)
                report_h5_name = os.path.join(dir_name, 'report_{}.h5'.format(hash(paths_str)))
                self._all_paths.append(report_h5_name)

                # check if cache is older than reports
                if os.path.exists(report_h5_name):
                    cache_mod_time = os.path.getmtime(report_h5_name)
                    regen_cache = False
                    for report_file in report_paths:
                        mod_time = os.path.getmtime(report_file)
                        if mod_time > cache_mod_time:
                            regen_cache = True
                    if regen_cache:
                        os.remove(report_h5_name)

                try:
                    # load dataframes from cache
                    self._reports_df = pandas.read_hdf(report_h5_name, 'report')
                    self._app_reports_df = pandas.read_hdf(report_h5_name, 'app_report')
                    if verbose:
                        sys.stdout.write('Loaded reports from {}.\n'.format(report_h5_name))
                except IOError as err:
                    sys.stderr.write('<geopmpy>: Warning: report HDF5 file not detected or older than reports.  Data will be saved to {}.\n'
                                     .format(report_h5_name))
                    self.parse_reports(report_paths, verbose)

                    # Cache report dataframe
                    try:
                        if verbose:
                            sys.stdout.write('Generating HDF5 files... ')
                        self._reports_df.to_hdf(report_h5_name, 'report', format='table')
                        self._app_reports_df.to_hdf(report_h5_name, 'app_report', format='table', append=True)
                    except ImportError as error:
                        sys.stderr.write('<geopmy> Warning: unable to write HDF5 file: {}\n'.format(str(error)))

                    if verbose:
                        sys.stdout.write('Done.\n')
                        sys.stdout.flush()
            else:
                self.parse_reports(report_paths, verbose)

        if traces:
            if type(traces) is str:
                trace_glob = os.path.join(dir_name, traces)
                trace_paths = natsorted(glob.glob(trace_glob))
                if len(trace_paths) == 0:
                    raise RuntimeError('No trace files found with pattern {}.'.format(trace_glob))
            elif type(traces) is list:
{'fraction_non_alphanumeric': 0.0697072838665759, 'fraction_numerical': 0.006603131381892444, 'mean_word_length': 2.209043250327654, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2447307', 'n_tokens_mistral': 4060, 'n_tokens_neox': 3727, 'n_words': 1406}