<commit_before>from .base import Vocabulary
import re

_hcpcs_split_regex = re.compile('([A-z]*)([0-9]+)')
def ubrev_split(code):
    match = _hcpcs_split_regex.match(code)
    letter_part = match.groups()[0]
    number_part = match.groups()[1]
    return letter_part, number_part

def ubrev_join(letter_part, number_part):
    digits = 5 - len(letter_part)
    return letter_part + (('%%.%dd' % digits) % int(number_part))



class HCPCS(Vocabulary):
    @staticmethod
    def _fill_range(start, end):
        start_letter, start_number = ubrev_split(start)
        end_letter, end_number = ubrev_split(end)
        assert start_letter == end_letter
        result = []
        for num in range(int(start_number), int(end_number) + 1):
            result.append(ubrev_join(start_letter, num))
        return result
    
    @staticmethod
    def _match_pattern(pattern):
        return [pattern]
    
    @staticmethod
    def standardize(code):
        result = code.strip()
        assert len(result) == 5<commit_msg>Fix names of split and join functions in hcpcs<commit_after>from .base import Vocabulary
import re

_hcpcs_split_regex = re.compile('([A-z]*)([0-9]+)')
def hcpcs_split(code):
    match = _hcpcs_split_regex.match(code)
    letter_part = match.groups()[0]
    number_part = match.groups()[1]
    return letter_part, number_part

def hcpcs_join(letter_part, number_part):
    digits = 5 - len(letter_part)
    return letter_part + (('%%.%dd' % digits) % int(number_part))



class HCPCS(Vocabulary):
    @staticmethod
    def _fill_range(start, end):
        start_letter, start_number = hcpcs_split(start)
        end_letter, end_number = hcpcs_split(end)
        assert start_letter == end_letter
        result = []
        for num in range(int(start_number), int(end_number) + 1):
            result.append(hcpcs_join(start_letter, num))
        return result
    
    @staticmethod
    def _match_pattern(pattern):
        return [pattern]
    
    @staticmethod
    def standardize(code):
        result = code.strip()
        assert len(result) == 5
{'fraction_non_alphanumeric': 0.11480945489628558, 'fraction_numerical': 0.006753497346840328, 'mean_word_length': 3.18989898989899, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6225699', 'n_tokens_mistral': 700, 'n_tokens_neox': 688, 'n_words': 145}