<commit_before>'''Several functions for generating subets of sigma star (in list form). Namely:
sigma_k, sigma_0k, sigma_1k each of which takes an alphabet and a value for k, and returns a list.
'''

def sigma_k(alphabet, k):
    '''Returns all words in sigma-k
    '''
    def helper(i):
        if i == 0:
            sigma_i = ['']
        else:
            sigma_i = []
            sigma_h = helper(i - 1)
            for word in sigma_h:
                sigma_i += [word + a for a in alphabet]
        return sigma_i
    return helper(k)

def sigma_0k(alphabet, k):
    '''Returns all words in all sigma-n from n = 0 to k.
    '''
    assert k >= 0
    def helper(i):
        language = ['']
        if i == 0:
            sigma_i = ['']
        else:
            sigma_i = []
            sigma_h, language = helper(i - 1)
            for word in sigma_h:
                new_stuff = [word + a for a in alphabet]
                sigma_i += new_stuff
                language += new_stuff
        return (sigma_i, language)
    return helper(k)[1]

def sigma_1k(alphabet, k):
    '''Return all words in all sigma-n from n = 1 to k.
    '''
    language = sigma_0k(alphabet, k)
    del language[0] # Get rid of empty string
    return language

# Example alphabet
cv = ['C','V']

# Example language
cv6 = sigma_1k(cv,6)<commit_msg>gen now produces strings with spaces.<commit_after>'''Several functions for generating subets of sigma star (in list form). Namely:
sigma_k, sigma_0k, sigma_1k each of which takes an alphabet and a value for k, and returns a list.
'''

def sigma_k(alphabet, k):
    '''Returns all words in sigma-k
    '''
    def helper(i):
        if i == 0:
            sigma_i = ['']
        else:
            sigma_i = []
            sigma_h = helper(i - 1)
            for word in sigma_h:
                if i > 1:
                    word += ' '
                sigma_i += [word + a for a in alphabet]
        return sigma_i
    return helper(k)

def sigma_0k(alphabet, k):
    '''Returns all words in all sigma-n from n = 0 to k.
    '''
    assert k >= 0
    def helper(i):
        language = ['']
        if i == 0:
            sigma_i = ['']
        else:
            sigma_i = []
            sigma_h, language = helper(i - 1)
            for word in sigma_h:
                if i > 1:
                    word += ' '
                new_stuff = [word + a for a in alphabet]
                sigma_i += new_stuff
                language += new_stuff
        return (sigma_i, language)
    return helper(k)[1]

def sigma_1k(alphabet, k):
    '''Return all words in all sigma-n from n = 1 to k.
    '''
    language = sigma_0k(alphabet, k)
    del language[0] # Get rid of empty string
    return language

# Example alphabet
cv = ['C','V']

# Example language
cv6 = sigma_1k(cv,6)
{'fraction_non_alphanumeric': 0.10790598290598291, 'fraction_numerical': 0.01282051282051282, 'mean_word_length': 1.7245392822502426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3640570', 'n_tokens_mistral': 950, 'n_tokens_neox': 856, 'n_words': 308}