<commit_before># -*- coding: utf-8 -*-

"""
genres.db
---

This module handles the genres database.
"""

import os


class Db:
    reference = None
    genres = None
    tags = None
    points = None

    def __init__(self, data_path=None):
        if not data_path:
            data_path = "%s/data.txt" % os.path.dirname(
                os.path.abspath(__file__))

        data = self.load(data_path)
        self.parse(data)

    def load(self, data_path):
        """
        Extract data from provided file and return it as a string.
        """
        with open(data_path, "r") as data_file:
            raw_data = data_file.read()

        data_file.close()
        return raw_data

    def parse(self, data):
        """
        Split and iterate through the datafile to extract genres, tags
        and points.
        """

        categories = data.split("\n\n")
        reference = {}
        reference_points = {}
        genre_index = []
        tag_index = []

        for category in categories:
            entries = category.strip().split("\n")
            entry_category, entry_points = self._parse_entry(entries[0].lower())

            if entry_category.startswith("#"):
                continue

            for entry in entries:
                entry = entry.lower()
                if not entry:
                    continue

                # Comment, ignore
                if entry.startswith("#"):
                    continue

                # Handle genre
                if not entry.startswith("-"):
                    genre, points = self._parse_entry(entry)

                    reference[genre] = entry_category
                    reference_points[genre] = points
                    genre_index.append(genre)

                # Handle tag
                else:
                    tag = entry[1:]
                    tag, points = self._parse_entry(tag)

                    reference[tag] = entry_category
                    reference_points[tag] = points
                    tag_index.append(tag)

        self.reference = reference
        self.genres = genre_index
        self.tags = tag_index
        self.points = reference_points

    @staticmethod
    def _parse_entry(entry, limit=10):
        """
        Finds both label and if provided, the points for ranking.
        """

        entry = entry.split(",")
        label = entry[0]
        points = limit

        if len(entry) > 1:
            proc = float(entry[1].strip())<commit_msg>Updated so tags are scored less<commit_after># -*- coding: utf-8 -*-

"""
genres.db
---

This module handles the genres database.
"""

import os


class Db:
    reference = None
    genres = None
    tags = None
    points = None

    def __init__(self, data_path=None):
        if not data_path:
            data_path = "%s/data.txt" % os.path.dirname(
                os.path.abspath(__file__))

        data = self.load(data_path)
        self.parse(data)

    def load(self, data_path):
        """
        Extract data from provided file and return it as a string.
        """
        with open(data_path, "r") as data_file:
            raw_data = data_file.read()

        data_file.close()
        return raw_data

    def parse(self, data):
        """
        Split and iterate through the datafile to extract genres, tags
        and points.
        """

        categories = data.split("\n\n")
        reference = {}
        reference_points = {}
        genre_index = []
        tag_index = []

        for category in categories:
            entries = category.strip().split("\n")
            entry_category, entry_points = self._parse_entry(entries[0].lower())

            if entry_category.startswith("#"):
                continue

            for entry in entries:
                entry = entry.lower()
                if not entry:
                    continue

                # Comment, ignore
                if entry.startswith("#"):
                    continue

                # Handle genre
                if not entry.startswith("-"):
                    genre, points = self._parse_entry(entry)

                    reference[genre] = entry_category
                    reference_points[genre] = points
                    genre_index.append(genre)

                # Handle tag
                else:
                    tag = entry[1:]
                    tag, points = self._parse_entry(tag, limit=9.5)

                    reference[tag] = entry_category
                    reference_points[tag] = points
                    tag_index.append(tag)

        self.reference = reference
        self.genres = genre_index
        self.tags = tag_index
        self.points = reference_points

    @staticmethod
    def _parse_entry(entry, limit=10):
        """
        Finds both label and if provided, the points for ranking.
        """

        entry = entry.split(",")
        label = entry[0]
        points = limit

        if len(entry) > 1:
            proc = float(entry[1].strip())
{'fraction_non_alphanumeric': 0.09718619038116144, 'fraction_numerical': 0.003592097385751347, 'mean_word_length': 1.4377431906614786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2790621', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1287, 'n_words': 337}