<commit_before>

import unicodedata

from . import BibliographySource, Reference
from . import Pages, Name, Date, DateRange, LiteralDate
from ..string import String, MixedString, NoCase
from .. import NAMES, DATES, NUMBERS



class CiteProcJSON(BibliographySource):
    def __init__(self, json_data):
        for ref in json_data:
            ref_data = {}
            for key, value in ref.items():
                python_key = key.replace('-', '_')
                if python_key == 'id':
                    ref_key = str(value).lower()
                    continue
                if python_key == 'type':
                    ref_type = value
                    continue
                elif python_key == 'key':
                    # conflicts with the ref_key, so ignore
                    continue
                elif python_key == 'page':
                    value = self.parse_page(value)
                elif python_key in NAMES:
                    value = self.parse_names(value)
                elif python_key in DATES:
                    value = self.parse_date(value)
                elif python_key == 'shortTitle':
                    python_key = 'title_short'
                else:
                    value = self.parse_string(value)

                ref_data[python_key] = value
            self.add(Reference(ref_key, ref_type, **ref_data))

    start_tag = '<span class="nocase">'
    end_tag = '</span>'

    def parse_string(self, string):
        string = str(string)
        lower_string = string.lower()
        end = 0
        output = MixedString()<commit_msg>Some entries should not be parsed as a MixedString<commit_after>

import unicodedata

from . import BibliographySource, Reference
from . import Pages, Name, Date, DateRange, LiteralDate
from ..string import String, MixedString, NoCase
from .. import NAMES, DATES, NUMBERS



class CiteProcJSON(BibliographySource):
    def __init__(self, json_data):
        for ref in json_data:
            ref_data = {}
            for key, value in ref.items():
                python_key = key.replace('-', '_')
                if python_key == 'id':
                    ref_key = str(value).lower()
                    continue
                elif python_key == 'type':
                    ref_type = value
                    continue
                elif python_key == 'key':
                    # conflicts with the ref_key, so ignore
                    continue
                if python_key == 'shortTitle':
                    python_key = 'title_short'

                if python_key == 'page':
                    value = self.parse_page(value)
                elif python_key in NAMES:
                    value = self.parse_names(value)
                elif python_key in DATES:
                    value = self.parse_date(value)
                elif key not in ('language', ):
                    value = self.parse_string(value)

                ref_data[python_key] = value
            self.add(Reference(ref_key, ref_type, **ref_data))

    start_tag = '<span class="nocase">'
    end_tag = '</span>'

    def parse_string(self, string):
        string = str(string)
        lower_string = string.lower()
        end = 0
        output = MixedString()
{'fraction_non_alphanumeric': 0.08830769230769231, 'fraction_numerical': 0.0006153846153846154, 'mean_word_length': 1.5378610460577673, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1322602', 'n_tokens_mistral': 894, 'n_tokens_neox': 819, 'n_words': 231}