<commit_before>var assert = require('assert')
var fs = require('fs')
var sgf = require('../modules/sgf')
var gametree = require('../modules/gametree')

describe('sgf', function() {
    describe('tokenize', function() {
        it('should work on multiple nodes', function() {
            assert.deepEqual(sgf.tokenize('(;B[aa]SZ[19];AB[cc][dd:ee])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'B'], ['c_value_type', '[aa]'],
                ['prop_ident', 'SZ'], ['c_value_type', '[19]'],
                ['semicolon', ';'],
                ['prop_ident', 'AB'], ['c_value_type', '[cc]'], ['c_value_type', '[dd:ee]'],
                ['parenthesis', ')']
            ])
        })
        it('should take escaping values into account', function() {
            assert.deepEqual(sgf.tokenize('(;C[hello\\]world];C[hello\\\\];C[hello])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[hello\\]world]'],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[hello\\\\]'],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[hello]'],
                ['parenthesis', ')']
            ])
            assert.deepEqual(sgf.tokenize('(;C[\\];B[aa];W[bb])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[\\];B[aa]'],
                ['semicolon', ';'],
                ['prop_ident', 'W'], ['c_value_type', '[bb]'],
                ['parenthesis', ')']
            ])
        })
        it('should allow lower case properties', function() {
            assert.deepEqual(sgf.tokenize('(;CoPyright[blah])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'CoPyright'], ['c_value_type', '[blah]'],
                ['parenthesis', ')']
            ])
        })
    })

    describe('parse', function() {
        it('should parse multiple nodes', function() {
            assert.equal(gametree.getJson(sgf.parse([
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'B'], ['c_value_type', '[aa]'],
                ['prop_ident', 'SZ'], ['c_value_type', '[19]'],
                ['semicolon', ';'],
                ['prop_ident', 'AB'], ['c_value_type', '[cc]'], ['c_value_type', '[dd:ee]'],
                ['parenthesis', ')']
            ])), gametree.getJson({
                nodes: [],
                subtrees: [
                    {
                        nodes: [
                            { B: ['aa'], SZ: ['19'] },
                            { AB: ['cc', 'dd:ee'] }
                        ],
                        subtrees: []
                    }
                ]
            }))
        })
        it('should parse variations', function() {
            assert.equal(
                gametree.getJson(sgf.parse(sgf.tokenize('(;B[hh](;W[ii])(;W[hi]C[h]))'))),
                gametree.getJson({
                    nodes: [],
                    subtrees: [
                        {
                            nodes: [{ B: ['hh'] }],
                            subtrees: [
                                {
                                    nodes: [{ W: ['ii'] }],
                                    subtrees: []
                                },
                                {
                                    nodes: [{ W: ['hi'], C: ['h'] }],
                                    subtrees: []
                                }
                            ]
                        }
                    ]
                })
            )
        })
        it('should convert lower case properties', function() {
            assert.equal(
                gametree.getJson(sgf.parse(sgf.tokenize('(;CoPyright[hello](;White[ii])(;White[hi]Comment[h]))'))),
                gametree.getJson({
                    nodes: [],
                    subtrees: [
                        {
                            nodes: [{ CP: ['hello'] }],
                            subtrees: [
                                {
                                    nodes: [{ W: ['ii'] }],
                                    subtrees: []
                                },
                                {
                                    nodes: [{ W: ['hi'], C: ['h'] }],
                                    subtrees: []
                                }
                            ]
                        }
                    ]
                })
            )
        })
        it('should parse a relatively complex file', function() {
            var contents = fs.readFileSync(__dirname + '/complex.sgf', 'utf8')
            var tokens = sgf.tokenize(contents)
            var tree = sgf.parse(tokens)

            assert.equal(tree.subtrees.length, 1)
        })
        it('should ignore empty subtrees', function() {
            assert.equal(
                gametree.getJson(sgf.parse(sgf.tokenize('(;B[hh]()(;W[ii])()(;W[hi]C[h]))'))),
                gametree.getJson({
                    nodes: [],
                    subtrees: [
                        {
                            nodes: [{ B: ['hh'] }],
                            subtrees: [
                                {
                                    nodes: [{ W: ['ii'] }],
                                    subtrees: []
                                },
                                {
                                    nodes: [{ W: ['hi'], C: ['h'] }],
                                    subtrees: []
                                }
                            ]
                        }
                    ]
                })
            )
        })
    })

    describe('string2dates', function() {
        it('should parse comma-separated dates', function() {
            assert.deepEqual(sgf.string2dates('1996-12-27,1997-01-03'), [
                [1996, 12, 27], [1997, 1, 3]
            ])
        })
        it('should be able to handle empty strings', function() {
            assert.deepEqual(sgf.string2dates(''), [])
        })
        it('should handle short-hand notation', function() {
            assert.deepEqual(sgf.string2dates('1996-05,06'), [
                [1996, 5], [1996, 6]
            ])
            assert.deepEqual(sgf.string2dates('1996-05-06,07,08'), [
                [1996, 5, 6], [1996, 5, 7], [1996, 5, 8]
            ])
            assert.deepEqual(sgf.string2dates('1996,1997'), [
                [1996], [1997]
            ])
            assert.deepEqual(sgf.string2dates('1996-12-27,28,1997-01-03,04'), [
                [1996, 12, 27], [1996, 12, 28], [1997, 1, 3], [1997, 1, 4]
            ])
        })
    })

    describe('dates2string', function() {
        it('should work', function() {
            assert.equal(sgf.dates2string([
                [1996, 5], [1996, 6]
            ]), '1996-05,06')
            assert.equal(sgf.dates2string([
                [1996, 5, 6], [1996, 5, 7], [1996, 5, 8]
            ]), '1996-05-06,07,08')
            assert.equal(sgf.dates2string([
                [1996], [1997]
            ]), '1996,1997')
            assert.equal(sgf.dates2string([
                [1996, 12, 27], [1996, 12, 28], [1997, 1, 3], [1997, 1, 4]<commit_msg>Add tests for sgf dates<commit_after>var assert = require('assert')
var fs = require('fs')
var sgf = require('../modules/sgf')
var gametree = require('../modules/gametree')

describe('sgf', function() {
    describe('tokenize', function() {
        it('should work on multiple nodes', function() {
            assert.deepEqual(sgf.tokenize('(;B[aa]SZ[19];AB[cc][dd:ee])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'B'], ['c_value_type', '[aa]'],
                ['prop_ident', 'SZ'], ['c_value_type', '[19]'],
                ['semicolon', ';'],
                ['prop_ident', 'AB'], ['c_value_type', '[cc]'], ['c_value_type', '[dd:ee]'],
                ['parenthesis', ')']
            ])
        })
        it('should take escaping values into account', function() {
            assert.deepEqual(sgf.tokenize('(;C[hello\\]world];C[hello\\\\];C[hello])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[hello\\]world]'],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[hello\\\\]'],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[hello]'],
                ['parenthesis', ')']
            ])
            assert.deepEqual(sgf.tokenize('(;C[\\];B[aa];W[bb])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'C'], ['c_value_type', '[\\];B[aa]'],
                ['semicolon', ';'],
                ['prop_ident', 'W'], ['c_value_type', '[bb]'],
                ['parenthesis', ')']
            ])
        })
        it('should allow lower case properties', function() {
            assert.deepEqual(sgf.tokenize('(;CoPyright[blah])'), [
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'CoPyright'], ['c_value_type', '[blah]'],
                ['parenthesis', ')']
            ])
        })
    })

    describe('parse', function() {
        it('should parse multiple nodes', function() {
            assert.equal(gametree.getJson(sgf.parse([
                ['parenthesis', '('],
                ['semicolon', ';'],
                ['prop_ident', 'B'], ['c_value_type', '[aa]'],
                ['prop_ident', 'SZ'], ['c_value_type', '[19]'],
                ['semicolon', ';'],
                ['prop_ident', 'AB'], ['c_value_type', '[cc]'], ['c_value_type', '[dd:ee]'],
                ['parenthesis', ')']
            ])), gametree.getJson({
                nodes: [],
                subtrees: [
                    {
                        nodes: [
                            { B: ['aa'], SZ: ['19'] },
                            { AB: ['cc', 'dd:ee'] }
                        ],
                        subtrees: []
                    }
                ]
            }))
        })
        it('should parse variations', function() {
            assert.equal(
                gametree.getJson(sgf.parse(sgf.tokenize('(;B[hh](;W[ii])(;W[hi]C[h]))'))),
                gametree.getJson({
                    nodes: [],
                    subtrees: [
                        {
                            nodes: [{ B: ['hh'] }],
                            subtrees: [
                                {
                                    nodes: [{ W: ['ii'] }],
                                    subtrees: []
                                },
                                {
                                    nodes: [{ W: ['hi'], C: ['h'] }],
                                    subtrees: []
                                }
                            ]
                        }
                    ]
                })
            )
        })
        it('should convert lower case properties', function() {
            assert.equal(
                gametree.getJson(sgf.parse(sgf.tokenize('(;CoPyright[hello](;White[ii])(;White[hi]Comment[h]))'))),
                gametree.getJson({
                    nodes: [],
                    subtrees: [
                        {
                            nodes: [{ CP: ['hello'] }],
                            subtrees: [
                                {
                                    nodes: [{ W: ['ii'] }],
                                    subtrees: []
                                },
                                {
                                    nodes: [{ W: ['hi'], C: ['h'] }],
                                    subtrees: []
                                }
                            ]
                        }
                    ]
                })
            )
        })
        it('should parse a relatively complex file', function() {
            var contents = fs.readFileSync(__dirname + '/complex.sgf', 'utf8')
            var tokens = sgf.tokenize(contents)
            var tree = sgf.parse(tokens)

            assert.equal(tree.subtrees.length, 1)
        })
        it('should ignore empty subtrees', function() {
            assert.equal(
                gametree.getJson(sgf.parse(sgf.tokenize('(;B[hh]()(;W[ii])()(;W[hi]C[h]))'))),
                gametree.getJson({
                    nodes: [],
                    subtrees: [
                        {
                            nodes: [{ B: ['hh'] }],
                            subtrees: [
                                {
                                    nodes: [{ W: ['ii'] }],
                                    subtrees: []
                                },
                                {
                                    nodes: [{ W: ['hi'], C: ['h'] }],
                                    subtrees: []
                                }
                            ]
                        }
                    ]
                })
            )
        })
    })

    describe('string2dates', function() {
        it('should parse comma-separated dates', function() {
            assert.deepEqual(sgf.string2dates('1996-12-27,1997-01-03'), [
                [1996, 12, 27], [1997, 1, 3]
            ])
        })
        it('should be able to handle empty strings', function() {
            assert.deepEqual(sgf.string2dates(''), [])
        })
        it('should handle short-hand notation', function() {
            assert.deepEqual(sgf.string2dates('1996-05,06'), [
                [1996, 5], [1996, 6]
            ])
            assert.deepEqual(sgf.string2dates('1996-05,06-01'), [
                [1996, 5], [1996, 6, 1]
            ])
            assert.deepEqual(sgf.string2dates('1996-05-06,07,08'), [
                [1996, 5, 6], [1996, 5, 7], [1996, 5, 8]
            ])
            assert.deepEqual(sgf.string2dates('1996,1997'), [
                [1996], [1997]
            ])
            assert.deepEqual(sgf.string2dates('1996-12-27,28,1997-01-03,04'), [
                [1996, 12, 27], [1996, 12, 28], [1997, 1, 3], [1997, 1, 4]
            ])
        })
    })

    describe('dates2string', function() {
        it('should work', function() {
            assert.equal(sgf.dates2string([
                [1996, 5], [1996, 6]
            ]), '1996-05,06')
            assert.equal(sgf.dates2string([
                [1996, 5], [1996, 6, 1]
            ]), '1996-05,06-01')
            assert.equal(sgf.dates2string([
                [1996, 5, 6], [1996, 5, 7], [1996, 5, 8]
            ]), '1996-05-06,07,08')
            assert.equal(sgf.dates2string([
                [1996], [1997]
            ]), '1996,1997')
            assert.equal(sgf.dates2string([
                [1996, 12, 27], [1996, 12, 28], [1997, 1, 3], [1997, 1, 4]
{'fraction_non_alphanumeric': 0.17603589337556083, 'fraction_numerical': 0.03668514119820533, 'mean_word_length': 1.0369573981991669, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2592700', 'n_tokens_mistral': 4706, 'n_tokens_neox': 4051, 'n_words': 658}