<commit_before># -*- coding: utf-8 -*-
from re import sub
from .templatetags.latex_escape import latex_escape

LATEX = {
    'verse_token': <KEY>',
    'chorus_token': '\\refren',
    'verse_end_token': '\koniec',
    'chorus_end_token': '\kon<PASSWORD>',
    'chords_token': '\chwyty',
}


class SongState:
    Nothing, Verse, Chorus = list(range(3))


def accidentals(str):
    return sub(
        r'([a-zA-Z]+)b',
        r'\1♭',
        sub(
            r'([a-zA-Z]+)#',
            r'\1♯',
            str(str)
        )
    )

def parse_song_content(song_content):
    result = []
    last_state = SongState.Nothing
    current_state = SongState.Nothing

    # Append last empty line to guarantee Nothing state on end.
    song_content += '\n'

    for line in song_content.split('\n'):
        last_state = current_state
        # Determining current state of the song
        is_empty = line.strip() == ''<commit_msg>parser: fix name conflict between argument and stdlib function<commit_after># -*- coding: utf-8 -*-
from re import sub
from .templatetags.latex_escape import latex_escape

LATEX = {
    'verse_token': <KEY>',
    'chorus_token': '\\<PASSWORD>',
    'verse_end_token': '\koniec',
    'chorus_end_token': '\kon<PASSWORD>',
    'chords_token': '\chwyty',
}


class SongState:
    Nothing, Verse, Chorus = list(range(3))


def accidentals(text):
    return sub(
        r'([a-zA-Z]+)b',
        r'\1♭',
        sub(
            r'([a-zA-Z]+)#',
            r'\1♯',
            str(text)
        )
    )

def parse_song_content(song_content):
    result = []
    last_state = SongState.Nothing
    current_state = SongState.Nothing

    # Append last empty line to guarantee Nothing state on end.
    song_content += '\n'

    for line in song_content.split('\n'):
        last_state = current_state
        # Determining current state of the song
        is_empty = line.strip() == ''
{'fraction_non_alphanumeric': 0.14540682414698164, 'fraction_numerical': 0.004199475065616798, 'mean_word_length': 2.471766848816029, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3670449', 'n_tokens_mistral': 654, 'n_tokens_neox': 650, 'n_words': 150}