<commit_before>import logging
import collections

from detectem.utils import (
    extract_version, extract_name, extract_version_from_headers,
    get_most_complete_version, check_presence
)
from detectem.plugin import get_plugin_by_name

logger = logging.getLogger('detectem')

VersionResult = collections.namedtuple('VersionResult', 'name version homepage')
IndicatorResult = collections.namedtuple('IndicatorResult', 'name homepage')


class Detector():
    def __init__(self, response, plugins, requested_url):
        self.har = response['har']
        self.requested_url = requested_url

        self._softwares = response['softwares']
        self._results = set()

        self.version_plugins = [p for p in plugins if not p.is_indicator]
        self.indicators = [p for p in plugins if p.is_indicator]

    def process_har(self):
        for entry in self.har:
            for plugin in self.version_plugins:
                version = self.get_plugin_version(plugin, entry)
                if version:
                    name = self.get_plugin_name(plugin, entry)
                    self._results.add(
                        VersionResult(name, version, plugin.homepage)
                    )

        # Feedback from Javascript
        for software in self._softwares:
            plugin = get_plugin_by_name(software['name'], self.version_plugins)
            self._results.add(
                VersionResult(plugin.name, software['version'], plugin.homepage)
            )

        for entry in self.har:
            for plugin in self.indicators:
                is_present = self.check_indicator_presence(plugin, entry)
                if is_present:
                    self._results.add(
                        IndicatorResult(plugin.name, plugin.homepage)
                    )

    def get_results(self, metadata=False):
        results_data = []

        self.process_har()
<commit_msg>Add hints support to core<commit_after>import logging
import collections

from detectem.utils import (
    extract_version, extract_name, extract_version_from_headers,
    get_most_complete_version, check_presence
)
from detectem.plugin import get_plugin_by_name

logger = logging.getLogger('detectem')

VersionResult = collections.namedtuple('VersionResult', 'name version homepage')
IndicatorResult = collections.namedtuple('IndicatorResult', 'name homepage')


class Detector():
    def __init__(self, response, plugins, requested_url):
        self.har = response['har']
        self.requested_url = requested_url

        self._softwares = response['softwares']
        self._results = set()

        # Separate plugins according to its type
        self.version_plugins = [p for p in plugins if not p.is_indicator]
        self.indicators = [p for p in plugins if p.is_indicator]

    def get_hints(self, plugin, entry):
        """ Get plugins hints from `plugin` on `entry`.

        Plugins hints could return `VersionResult`, `IndicatorResult` or `None`.

        """
        hints = []

        for hint_function in getattr(plugin, 'hints', []):
            hint = hint_function(entry)
            if hint:
                hints.append(hint)

        return hints

    def process_har(self):
        """ Detect plugins present in the page.

        First, start with version plugins, then software from Splash
        and finish with indicators.
        In each phase try to detect plugin hints in already detected plugins.

        """
        hints = []
        for entry in self.har:
            for plugin in self.version_plugins:
                version = self.get_plugin_version(plugin, entry)
                if version:
                    # Name can be different than plugin name in modular plugins
                    name = self.get_plugin_name(plugin, entry)
                    self._results.add(
                        VersionResult(name, version, plugin.homepage)
                    )
                    hints += self.get_hints(plugin, entry)

        # Feedback from Javascript, it comes from Splash
        for software in self._softwares:
            plugin = get_plugin_by_name(software['name'], self.version_plugins)
            self._results.add(
                VersionResult(plugin.name, software['version'], plugin.homepage)
            )
            hints += self.get_hints(plugin, entry)

        for entry in self.har:
            for plugin in self.indicators:
                is_present = self.check_indicator_presence(plugin, entry)
                if is_present:
                    self._results.add(
                        IndicatorResult(plugin.name, plugin.homepage)
                    )
                    import pudb; pudb.set_trace()
                    hints += self.get_hints(plugin, entry)

        for hint in hints:
            self._results.add(hint)

    def get_results(self, metadata=False):
        """ Return results of the analysis. """
        results_data = []

        self.process_har()

{'fraction_non_alphanumeric': 0.08130408532904004, 'fraction_numerical': 0.0, 'mean_word_length': 2.123821495914519, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '866947', 'n_tokens_mistral': 1335, 'n_tokens_neox': 1261, 'n_words': 342}