<commit_before># -*- coding: utf-8 -*-

"""
genres.finder
---

This module handles the genre finder.
"""

from collections import Counter
import re


class Finder():
    db = None
    unique_category = True

    def __init__(self, db, unique_category=True):
        self.db = db
        self.unique_category = unique_category

    def find(self, text):
        """
        Return a list of genres found in text.
        """

        genres = []
        text = text.lower()

        category_counter = Counter()
        counter = Counter()

        for genre in self.db.genres:
            found = self.contains_entity(genre, text)
            if found:
                counter[genre] += found

                category = self.db.reference[genre]
                points = self.db.points[genre]
                points *= found
                category_counter[category] += points

        for tag in self.db.tags:
            found = self.contains_entity(tag, text)
            if found:
                category = self.db.reference[tag]

                if not counter[category]:
                    counter[category] += found

                points = self.db.points[category]
                points *= found
                category_counter[category] += points

        if len(category_counter) == 0:
            return genres

        main_category = category_counter.most_common(1)[0][0]

        # Convert counter to a flat list of genres, sorted by count
        sorted_genres = [ite for ite, it in counter.most_common()]

        for genre in sorted_genres:
            insert = True

            if self.unique_category:
                if not self.db.reference[genre] == main_category:
                    insert = False

            if insert:
                genres.append(genre)

        return genres

    @staticmethod
    def contains_entity(entity, text):
        """
        Attempt to try entity, return false if not found. Otherwise the
        amount of time entitu is occuring.
        """

        try:<commit_msg>Fixed issue where tag did not use own points. Added additional points for frequency<commit_after># -*- coding: utf-8 -*-

"""
genres.finder
---

This module handles the genre finder.
"""

from collections import Counter
import re


class Finder():
    db = None
    unique_category = True

    def __init__(self, db, unique_category=True):
        self.db = db
        self.unique_category = unique_category

    def find(self, text):
        """
        Return a list of genres found in text.
        """

        genres = []
        text = text.lower()

        category_counter = Counter()
        counter = Counter()

        for genre in self.db.genres:
            found = self.contains_entity(genre, text)
            if found:
                counter[genre] += found

                category = self.db.reference[genre]
                points = self.db.points[genre]
                points *= found

                # Add bonus points if additional terms points to category
                if category_counter[category] > 0:
                    points += 1

                category_counter[category] += points

        for tag in self.db.tags:
            found = self.contains_entity(tag, text)
            if found:
                category = self.db.reference[tag]

                if not counter[category]:
                    counter[category] += found

                points = self.db.points[tag]
                points *= found
                category_counter[category] += points

        if len(category_counter) == 0:
            return genres

        main_category = category_counter.most_common(1)[0][0]

        # Convert counter to a flat list of genres, sorted by count
        sorted_genres = [ite for ite, it in counter.most_common()]

        for genre in sorted_genres:
            insert = True

            if self.unique_category:
                if not self.db.reference[genre] == main_category:
                    insert = False

            if insert:
                genres.append(genre)

        return genres

    @staticmethod
    def contains_entity(entity, text):
        """
        Attempt to try entity, return false if not found. Otherwise the
        amount of time entitu is occuring.
        """

        try:
{'fraction_non_alphanumeric': 0.08383791336748952, 'fraction_numerical': 0.0027945971122496508, 'mean_word_length': 1.6157125456760049, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5853098', 'n_tokens_mistral': 1156, 'n_tokens_neox': 1065, 'n_words': 336}