<commit_before># encoding: utf-8
# cython: profile=False, boundscheck=False, wraparound=False

from ._bp cimport BP
import time
import numpy as np
cimport numpy as np
cimport cython
np.import_array()


cdef void _set_node_metadata(np.uint32_t ptr, unicode token,
                             np.ndarray[object, ndim=1] names, 
                             np.ndarray[np.double_t, ndim=1] lengths):
    """Inplace update of names and lengths given token details"""
    cdef:
        np.double_t length
        Py_ssize_t split_idx, i
        unicode name

    name = None
    length = 0.0

    if token[0] == u':':
        length = np.double(token[1:])
    elif ':' in token:
        split_idx = token.rfind(':')
        name = token[:split_idx]
        length = np.double(token[split_idx + 1:])
        name = name.strip("'")
    else:
        name = token.replace("'", "").replace('"', "")
        pass

    names[ptr] = name
    lengths[ptr] = length


def write_newick(BP tree, object output):
    cdef:
        list name_stack
        list length_stack
        list open_paren_stack
        object name
        np.npy_float64 length
        Py_ssize_t idx
        np.npy_uint8 v
        Py_ssize_t root_close

    length_stack = []
    name_stack = []
    open_paren_stack = []
    root_close = tree.close(0)

    for idx, v in enumerate(tree.B):
        if v:
            if not tree.isleaf(idx):
                output.write('(')
            name_stack.append(tree.name(idx))
            length_stack.append(tree.length(idx))
            open_paren_stack.append(idx)
        else:
            name = name_stack.pop()
            length = length_stack.pop()

            if name is not None:
                # if we have magical characters, make sure we quote
                if set(name) & {';', ',', '(', ')'}:
                    output.write("'%s'" % name)
                else:
                    output.write(name)

            if length > 0:
                output.write(':%f' % length)
                
            if tree.nsibling(open_paren_stack.pop()) == 0:
                if idx != root_close:
                    output.write(')')
            else:
                output.write(',')
<commit_msg>addressing @kwcantrell's comment<commit_after># encoding: utf-8
# cython: profile=False, boundscheck=False, wraparound=False

from ._bp cimport BP
import time
import numpy as np
cimport numpy as np
cimport cython
np.import_array()


cdef void _set_node_metadata(np.uint32_t ptr, unicode token,
                             np.ndarray[object, ndim=1] names, 
                             np.ndarray[np.double_t, ndim=1] lengths):
    """Inplace update of names and lengths given token details"""
    cdef:
        np.double_t length
        Py_ssize_t split_idx, i
        unicode name

    name = None
    length = 0.0

    if token[0] == u':':
        length = np.double(token[1:])
    elif ':' in token:
        split_idx = token.rfind(':')
        name = token[:split_idx]
        length = np.double(token[split_idx + 1:])
        name = name.strip("'")
    else:
        name = token.replace("'", "").replace('"', "")
        pass

    names[ptr] = name
    lengths[ptr] = length


def write_newick(BP tree, object output):
    cdef:
        list name_stack
        list length_stack
        list open_paren_stack
        object name
        np.npy_float64 length
        Py_ssize_t idx
        np.npy_uint8 v
        Py_ssize_t root_close

    length_stack = []
    name_stack = []
    open_paren_stack = []
    root_close = tree.close(0)

    for idx, v in enumerate(tree.B):
        if v:
            if not tree.isleaf(idx):
                output.write('(')
            name_stack.append(tree.name(idx))
            length_stack.append(tree.length(idx))
            open_paren_stack.append(idx)
        else:
            name = name_stack.pop()
            length = length_stack.pop()

            if name is not None:
                # if we have magical characters, make sure we quote
                if set(name) & {';', ',', '(', ')', ':'}:
                    output.write("'%s'" % name)
                else:
                    output.write(name)

            if length > 0:
                output.write(':%f' % length)
                
            if tree.nsibling(open_paren_stack.pop()) == 0:
                if idx != root_close:
                    output.write(')')
            else:
                output.write(',')

{'fraction_non_alphanumeric': 0.11355475763016158, 'fraction_numerical': 0.00718132854578097, 'mean_word_length': 1.6688622754491018, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6902288', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1346, 'n_words': 328}