<commit_before>function strtok (str, tokens) {
    // http://kevin.vanzonneveld.net
    // +   original by: <NAME> (http://brettz9.blogspot.com)
    // %        note 1: Use tab and newline as tokenizing characters as well
    // *     example 1: $string = "\t\t\t\nThis is\tan example\nstring\n";
    // *     example 1: $tok = strtok($string, " \n\t");
    // *     example 1: $b = '';
    // *     example 1: while($tok !== false) {$b += "Word="+$tok+"\n"; $tok = strtok(" \n\t");}
    // *     example 1: $b
    // *     returns 1: "Word=This\nWord=is\nWord=an\nWord=example\nWord=string\n"

    if (tokens === undefined) {
        tokens = str;
        str = strtok.leftOver;
    }
    if (str.length === 0) {
        return false;
    }
    if (tokens.indexOf(str[0]) !== -1) {
        return strtok(str.substr(1), tokens);
    }
    for (var i=0; i < str.length; i++) {
        if (tokens.indexOf(str[i]) !== -1) {
            break;
        }
    }
    strtok.leftOver = str.substr(i+1);
    return str.substring(0, i);
}<commit_msg>Fix recursive reference to use 'this' and don't store tokenization info on function.<commit_after>function strtok (str, tokens) {
    // http://kevin.vanzonneveld.net
    // +   original by: <NAME> (http://brettz9.blogspot.com)
    // %        note 1: Use tab and newline as tokenizing characters as well
    // *     example 1: $string = "\t\t\t\nThis is\tan example\nstring\n";
    // *     example 1: $tok = strtok($string, " \n\t");
    // *     example 1: $b = '';
    // *     example 1: while($tok !== false) {$b += "Word="+$tok+"\n"; $tok = strtok(" \n\t");}
    // *     example 1: $b
    // *     returns 1: "Word=This\nWord=is\nWord=an\nWord=example\nWord=string\n"

    if (!this.php_js) {
        this.php_js = {};
    }

    if (tokens === undefined) {
        tokens = str;
        str = this.php_js.strtokleftOver;
    }
    if (str.length === 0) {
        return false;
    }
    if (tokens.indexOf(str[0]) !== -1) {
        return this.strtok(str.substr(1), tokens);
    }
    for (var i=0; i < str.length; i++) {
        if (tokens.indexOf(str[i]) !== -1) {
            break;
        }
    }
    this.php_js.strtokleftOver = str.substr(i+1);
    return str.substring(0, i);
}
{'fraction_non_alphanumeric': 0.19149888143176733, 'fraction_numerical': 0.014317673378076063, 'mean_word_length': 2.009421265141319, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3682853', 'n_tokens_mistral': 827, 'n_tokens_neox': 802, 'n_words': 179}