<commit_before>var Tuple = require('../lib/tuple')

var gametree = require('../module/gametree')
var fs = require('fs')

var alpha = 'abcdefghijklmnopqrstuvwxyz'

exports.meta = {
    name: 'Smart Game Format',
    extensions: ['sgf']
}

exports.tokenize = function(input) {
    var tokens = []
    var rules = {
        ignore: /^\s+/,
        parenthesis: /^(\(|\))/,
        semicolon: /^;/,
        prop_ident: /^[A-Za-z]+/,
        c_value_type: /^\[(\]|[^]*?[^\\]\])/
    }

    while (input.length > 0) {
        var token = null
        var length = 0

        for (var type in rules) {
            var matches = rules[type].exec(input)
            if (!matches) continue

            var value = matches[0]
            length = value.length

            if (type == 'c_value_type')
                value = exports.unescapeString(value.substr(1, length - 2))
            token = new Tuple(type, value)

            break
        }

        if (token && token[0] != 'ignore') tokens.push(token)
        input = input.substr(length)
    }

    return tokens
}

exports.parse = function(tokens, callback, start, depth) {
    if (!callback) callback = function(progress) {}
    if (!start) start = [0]
    if (isNaN(depth)) depth = 0

    var i = start[0]
    var node, property, tree = gametree.new()
    tree.collapsed = tokens.length >= setting.get('graph.collapse_tokens_count')
        && depth >= setting.get('graph.collapse_min_depth')<commit_msg>Ensure there is no infinite loop<commit_after>var Tuple = require('../lib/tuple')

var gametree = require('../module/gametree')
var fs = require('fs')

var alpha = 'abcdefghijklmnopqrstuvwxyz'

exports.meta = {
    name: 'Smart Game Format',
    extensions: ['sgf']
}

exports.tokenize = function(input) {
    var tokens = []
    var rules = {
        ignore: /^\s+/,
        parenthesis: /^(\(|\))/,
        semicolon: /^;/,
        prop_ident: /^[A-Za-z]+/,
        c_value_type: /^\[(\]|[^]*?[^\\]\])/
    }

    while (input.length > 0) {
        var token = null
        var length = 1

        for (var type in rules) {
            var matches = rules[type].exec(input)
            if (!matches) continue

            var value = matches[0]
            length = value.length

            if (type == 'c_value_type')
                value = exports.unescapeString(value.substr(1, length - 2))
            token = new Tuple(type, value)

            break
        }

        if (token && token[0] != 'ignore') tokens.push(token)
        input = input.substr(length)
    }

    return tokens
}

exports.parse = function(tokens, callback, start, depth) {
    if (!callback) callback = function(progress) {}
    if (!start) start = [0]
    if (isNaN(depth)) depth = 0

    var i = start[0]
    var node, property, tree = gametree.new()
    tree.collapsed = tokens.length >= setting.get('graph.collapse_tokens_count')
        && depth >= setting.get('graph.collapse_min_depth')
{'fraction_non_alphanumeric': 0.15741056218057922, 'fraction_numerical': 0.006132879045996593, 'mean_word_length': 2.1367521367521367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1012290', 'n_tokens_mistral': 967, 'n_tokens_neox': 929, 'n_words': 220}