<commit_before>describe "tokenize", -> describe "parenthesize", ->
    rewire = require "rewire"
    describe "imports", ->
        tokenizeParenthesize = rewire "./parenthesize"
        it "parentheses", -> (expect tokenizeParenthesize.__get__ "parentheses").toBe require "./parentheses"
        
    describe "on calling", ->
        tokenizeParenthesize = rewire "./parenthesize"
        
        parentheses = 
            "openera": "closera"
            "openerb": "closerb"
        
        run = (config) ->
            describe config.description, ->
                result = inputCopy = parenthesesCopy = undefined
                
                beforeEach ->
                    parenthesesCopy = JSON.parse JSON.stringify parentheses
                    tokenizeParenthesize.__set__ "parentheses", parenthesesCopy
                    
                    inputCopy = JSON.parse JSON.stringify config.input
                
                if config.output
                    it "returns the expected result", -> (expect tokenizeParenthesize inputCopy).toEqual config.output
                else
                    it "throws the expected exception", -> (expect -> tokenizeParenthesize inputCopy).toThrow config.throws

                beforeEach -> 
                    try
                        tokenizeParenthesize inputCopy
                    catch
                    
                it "does not change parentheses", -> (expect parenthesesCopy).toEqual parentheses
                it "does not modify the input", -> (expect inputCopy).toEqual config.input

        run 
            description: "nothing"
            input: []
            output: []
                
        run 
            description: "one non-parenthesis"
            input: [
                token: "misca"
                starts: 5
            ]
            output: [
                token: "misca"
                starts: 5
                ends: 9
            ]
            
        run 
            description: "multiple non-parentheses"
            input: [
                    token: "misca"
                    starts: 5
                ,
                    token: "miscb"
                    starts: 17
                ,
                    token: "miscc"
                    starts: 39
            ]
            output: [
                    token: "misca"
                    starts: 5
                    ends: 9
                ,
                    token: "miscb"
                    starts: 17
                    ends: 21
                ,
                    token: "miscc"
                    starts: 39
                    ends: 43
            ]
        
        run 
            description: "one opening parenthesis"
            input: [
                token: "openera"
                starts: 5
            ]
            throws:
                reason: "unclosedParentheses"
                starts: 5
                ends: 11
                    
        run <commit_msg>Added tests to ensure that tokenizeParenthesize does not pick up on JavaScript Object.prototype properties.<commit_after>describe "tokenize", -> describe "parenthesize", ->
    rewire = require "rewire"
    describe "imports", ->
        tokenizeParenthesize = rewire "./parenthesize"
        it "parentheses", -> (expect tokenizeParenthesize.__get__ "parentheses").toBe require "./parentheses"
        
    describe "on calling", ->
        tokenizeParenthesize = rewire "./parenthesize"
        
        parentheses = 
            "openera": "closera"
            "openerb": "closerb"
        
        run = (config) ->
            describe config.description, ->
                result = inputCopy = parenthesesCopy = undefined
                
                beforeEach ->
                    parenthesesCopy = JSON.parse JSON.stringify parentheses
                    tokenizeParenthesize.__set__ "parentheses", parenthesesCopy
                    
                    inputCopy = JSON.parse JSON.stringify config.input
                
                if config.output
                    it "returns the expected result", -> (expect tokenizeParenthesize inputCopy).toEqual config.output
                else
                    it "throws the expected exception", -> (expect -> tokenizeParenthesize inputCopy).toThrow config.throws

                beforeEach -> 
                    try
                        tokenizeParenthesize inputCopy
                    catch
                    
                it "does not change parentheses", -> (expect parenthesesCopy).toEqual parentheses
                it "does not modify the input", -> (expect inputCopy).toEqual config.input

        run 
            description: "nothing"
            input: []
            output: []
                
        run 
            description: "one non-parenthesis"
            input: [
                token: "misca"
                starts: 5
            ]
            output: [
                token: "misca"
                starts: 5
                ends: 9
            ]
            
        run 
            description: "multiple non-parentheses"
            input: [
                    token: "misca"
                    starts: 5
                ,
                    token: "miscb"
                    starts: 17
                ,
                    token: "miscc"
                    starts: 39
            ]
            output: [
                    token: "misca"
                    starts: 5
                    ends: 9
                ,
                    token: "miscb"
                    starts: 17
                    ends: 21
                ,
                    token: "miscc"
                    starts: 39
                    ends: 43
            ]
            
        run 
            description: "ignores prototype keys"
            input: [
                    token: "misca"
                    starts: 5
                ,
                    token: "constructor"
                    starts: 17
                ,
                    token: "miscc"
                    starts: 39
            ]
            output: [
                    token: "misca"
                    starts: 5
                    ends: 9
                ,
                    token: "constructor"
                    starts: 17
                    ends: 27
                ,
                    token: "miscc"
                    starts: 39
                    ends: 43
            ]
        
        run 
            description: "one opening parenthesis"
            input: [
                token: "openera"
                starts: 5
            ]
            throws:
                reason: "unclosedParentheses"
                starts: 5
                ends: 11
                    
        run 
{'fraction_non_alphanumeric': 0.06866067481950788, 'fraction_numerical': 0.008693089730366877, 'mean_word_length': 0.8818963127252565, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7579775', 'n_tokens_mistral': 1656, 'n_tokens_neox': 1430, 'n_words': 386}