<commit_before>#!/usr/bin/env python

"""
CmpRuns - A simple tool for comparing two static analyzer runs to determine
which reports have been added, removed, or changed.

This is designed to support automated testing using the static analyzer, from
two perspectives:
  1. To monitor changes in the static analyzer's reports on real code bases,
     for regression testing.

  2. For use by end users who want to integrate regular static analyzer testing
     into a buildbot like environment.

Usage:

    # Load the results of both runs, to obtain lists of the corresponding
    # AnalysisDiagnostic objects.
    #
    resultsA = loadResultsFromSingleRun(singleRunInfoA, deleteEmpty)
    resultsB = loadResultsFromSingleRun(singleRunInfoB, deleteEmpty)

    # Generate a relation from diagnostics in run A to diagnostics in run B
    # to obtain a list of triples (a, b, confidence).
    diff = compareResults(resultsA, resultsB)

"""
from __future__ import division, print_function

from collections import defaultdict

from math import log
from optparse import OptionParser
import json
import os
import plistlib
import re
import sys

STATS_REGEXP = re.compile(r"Statistics: (\{.+\})", re.MULTILINE | re.DOTALL)

class Colors(object):
    """
    Color for terminal highlight.
    """
    RED = '\x1b[2;30;41m'
    GREEN = '\x1b[6;30;42m'
    CLEAR = '\x1b[0m'

# Information about analysis run:
# path - the analysis output directory
# root - the name of the root directory, which will be disregarded when
# determining the source file name
class SingleRunInfo(object):
    def __init__(self, path, root="", verboseLog=None):
        self.path = path
        self.root = root.rstrip("/\\")
        self.verboseLog = verboseLog


class AnalysisDiagnostic(object):
    def __init__(self, data, report, htmlReport):
        self._data = data
        self._loc = self._data['location']
        self._report = report
        self._htmlReport = htmlReport
        self._reportSize = len(self._data['path'])

    def getFileName(self):
        root = self._report.run.root
        fileName = self._report.files[self._loc['file']]
        if fileName.startswith(root) and len(root) > 0:
            return fileName[len(root) + 1:]
        return fileName

    def getLine(self):
        return self._loc['line']

    def getColumn(self):
        return self._loc['col']

    def getPathLength(self):
        return self._reportSize

    def getCategory(self):
        return self._data['category']

    def getDescription(self):
        return self._data['description']

    def getIssueIdentifier(self):
        id = self.getFileName() + "+"
        if 'issue_context' in self._data:
            id += self._data['issue_context'] + "+"
        if 'issue_hash_content_of_line_in_context' in self._data:
            id += str(self._data['issue_hash_content_of_line_in_context'])
        return id

    def getReport(self):
        if self._htmlReport is None:
            return " "
        return os.path.join(self._report.run.path, self._htmlReport)

    def getReadableName(self):
        if 'issue_context' in self._data:
            funcnamePostfix = "#" + self._data['issue_context']
        else:
            funcnamePostfix = ""
        return '%s%s:%d:%d, %s: %s' % (self.getFileName(),
                                       funcnamePostfix,
                                       self.getLine(),
                                       self.getColumn(), self.getCategory(),
                                       self.getDescription())

    # Note, the data format is not an API and may change from one analyzer
    # version to another.
    def getRawData(self):
        return self._data<commit_msg>[analyzer] [testing] Inside CmpRuns.py output also print the filename of the first item in the path<commit_after>#!/usr/bin/env python

"""
CmpRuns - A simple tool for comparing two static analyzer runs to determine
which reports have been added, removed, or changed.

This is designed to support automated testing using the static analyzer, from
two perspectives:
  1. To monitor changes in the static analyzer's reports on real code bases,
     for regression testing.

  2. For use by end users who want to integrate regular static analyzer testing
     into a buildbot like environment.

Usage:

    # Load the results of both runs, to obtain lists of the corresponding
    # AnalysisDiagnostic objects.
    #
    resultsA = loadResultsFromSingleRun(singleRunInfoA, deleteEmpty)
    resultsB = loadResultsFromSingleRun(singleRunInfoB, deleteEmpty)

    # Generate a relation from diagnostics in run A to diagnostics in run B
    # to obtain a list of triples (a, b, confidence).
    diff = compareResults(resultsA, resultsB)

"""
from __future__ import division, print_function

from collections import defaultdict

from math import log
from optparse import OptionParser
import json
import os
import plistlib
import re
import sys

STATS_REGEXP = re.compile(r"Statistics: (\{.+\})", re.MULTILINE | re.DOTALL)

class Colors(object):
    """
    Color for terminal highlight.
    """
    RED = '\x1b[2;30;41m'
    GREEN = '\x1b[6;30;42m'
    CLEAR = '\x1b[0m'

# Information about analysis run:
# path - the analysis output directory
# root - the name of the root directory, which will be disregarded when
# determining the source file name
class SingleRunInfo(object):
    def __init__(self, path, root="", verboseLog=None):
        self.path = path
        self.root = root.rstrip("/\\")
        self.verboseLog = verboseLog


class AnalysisDiagnostic(object):
    def __init__(self, data, report, htmlReport):
        self._data = data
        self._loc = self._data['location']
        self._report = report
        self._htmlReport = htmlReport
        self._reportSize = len(self._data['path'])

    def getFileName(self):
        root = self._report.run.root
        fileName = self._report.files[self._loc['file']]
        if fileName.startswith(root) and len(root) > 0:
            return fileName[len(root) + 1:]
        return fileName

    def getRootFileName(self):
        path = self._data['path']
        if not path:
            return self.getFileName()
        p = path[0]
        if 'location' in p:
            fIdx = p['location']['file']
        else: # control edge
            fIdx = path[0]['edges'][0]['start'][0]['file']
        out = self._report.files[fIdx]
        root = self._report.run.root
        if out.startswith(root):
            return out[len(root):]
        return out

    def getLine(self):
        return self._loc['line']

    def getColumn(self):
        return self._loc['col']

    def getPathLength(self):
        return self._reportSize

    def getCategory(self):
        return self._data['category']

    def getDescription(self):
        return self._data['description']

    def getIssueIdentifier(self):
        id = self.getFileName() + "+"
        if 'issue_context' in self._data:
            id += self._data['issue_context'] + "+"
        if 'issue_hash_content_of_line_in_context' in self._data:
            id += str(self._data['issue_hash_content_of_line_in_context'])
        return id

    def getReport(self):
        if self._htmlReport is None:
            return " "
        return os.path.join(self._report.run.path, self._htmlReport)

    def getReadableName(self):
        if 'issue_context' in self._data:
            funcnamePostfix = "#" + self._data['issue_context']
        else:
            funcnamePostfix = ""
        rootFilename = self.getRootFileName()
        fileName = self.getFileName()
        if rootFilename != fileName:
            filePrefix = "[%s] %s" % (rootFilename, fileName)
        else:
            filePrefix = rootFilename
        return '%s%s:%d:%d, %s: %s' % (filePrefix,
                                       funcnamePostfix,
                                       self.getLine(),
                                       self.getColumn(), self.getCategory(),
                                       self.getDescription())

    # Note, the data format is not an API and may change from one analyzer
    # version to another.
    def getRawData(self):
        return self._data
{'fraction_non_alphanumeric': 0.0970933072789448, 'fraction_numerical': 0.004885197850512946, 'mean_word_length': 2.4950917626973963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2819488', 'n_tokens_mistral': 2319, 'n_tokens_neox': 2243, 'n_words': 703}