<commit_before>function markdown(string = '', marks) {
    if (marks == null) {
        return [{
            mark: null,
            text: string
        }]
    }

    let offset = 0,
        count = 0,
        regex = new RegExp(marks
            .map(a => '\\' + a)
            .join('|')
        );

    return string
        .split(regex)
        .map(text => {
            let index = offset - 1,
                mark = null;

            if (index > -1) {
                if (count % 2 === 0) {
                    mark = string[index];
                }
                count += 1;
            }
            offset += text.length + 1;

            if (text !== '') {
                return {
                    mark,
                    text
                }
            }
        })
        .filter(a => a);
}

export default markdown;<commit_msg>Update markdown.js<commit_after>function markdown(string = '', marks) {
    if (marks == null) {
        return [{
            mark: null,
            text: string
        }]
    }
    let offset = 0,
        count = 0,
        regex = new RegExp(marks
            .map(a => '\\' + a)
            .join('|')
        );
    return string
        .split(regex)
        .map(text => {
            let index = offset - 1,
                mark = null;

            if (index > -1) {
                if (count % 2 === 0) {
                    mark = string[index];
                }
                count += 1;
            }
            offset += text.length + 1;

            if (text !== '') {
                return {
                    mark,
                    text
                }
            }
        })
        .filter(a => a);
}

export default markdown;

{'fraction_non_alphanumeric': 0.12616279069767442, 'fraction_numerical': 0.009302325581395349, 'mean_word_length': 0.8115789473684211, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6693416', 'n_tokens_mistral': 494, 'n_tokens_neox': 452, 'n_words': 112}