<commit_before>syntax = (require 'esprima').Syntax
debug = require 'debug'

_ArrayExpression = syntax.ArrayExpression
_ArrayPattern = syntax.ArrayPattern
_ArrowFunctionExpression = syntax.ArrowFunctionExpression
_AssignmentExpression = syntax.AssignmentExpression
_BinaryExpression = syntax.BinaryExpression
_BlockStatement = syntax.BlockStatement
_BreakStatement = syntax.BreakStatement
_CallExpression = syntax.CallExpression
_CatchClause = syntax.CatchClause
_ClassBody = syntax.ClassBody
_ClassDeclaration = syntax.ClassDeclaration
_ClassExpression = syntax.ClassExpression
_ClassHeritage = syntax.ClassHeritage
_ComprehensionBlock = syntax.ComprehensionBlock
_ComprehensionExpression = syntax.ComprehensionExpression
_ConditionalExpression = syntax.ConditionalExpression
_ContinueStatement = syntax.ContinueStatement
_DebuggerStatement = syntax.DebuggerStatement
_DoWhileStatement = syntax.DoWhileStatement
_EmptyStatement = syntax.EmptyStatement
_ExportDeclaration = syntax.ExportDeclaration
_ExportBatchSpecifier = syntax.ExportBatchSpecifier
_ExportSpecifier = syntax.ExportSpecifier
_ExpressionStatement = syntax.ExpressionStatement
_ForInStatement = syntax.ForInStatement
_ForOfStatement = syntax.ForOfStatement
_ForStatement = syntax.ForStatement
_FunctionDeclaration = syntax.FunctionDeclaration
_FunctionExpression = syntax.FunctionExpression
_Identifier = syntax.Identifier
_IfStatement = syntax.IfStatement
_ImportDeclaration = syntax.ImportDeclaration
_ImportSpecifier = syntax.ImportSpecifier
_LabeledStatement = syntax.LabeledStatement
_Literal = syntax.Literal
_LogicalExpression = syntax.LogicalExpression
_MemberExpression = syntax.MemberExpression
_MethodDefinition = syntax.MethodDefinition
_ModuleDeclaration = syntax.ModuleDeclaration
_NewExpression = syntax.NewExpression
_ObjectExpression = syntax.ObjectExpression
_ObjectPattern = syntax.ObjectPattern
_Program = syntax.Program
_Property = syntax.Property
_ReturnStatement = syntax.ReturnStatement
_SequenceExpression = syntax.SequenceExpression
_SpreadElement = syntax.SpreadElement
_SwitchCase = syntax.SwitchCase
_SwitchStatement = syntax.SwitchStatement
_TaggedTemplateExpression = syntax.TaggedTemplateExpression
_TemplateElement = syntax.TemplateElement
_TemplateLiteral = syntax.TemplateLiteral
_ThisExpression = syntax.ThisExpression
_ThrowStatement = syntax.ThrowStatement
_TryStatement = syntax.TryStatement
_UnaryExpression = syntax.UnaryExpression
_UpdateExpression = syntax.UpdateExpression
_VariableDeclaration = syntax.VariableDeclaration
_VariableDeclarator = syntax.VariableDeclarator
_WhileStatement = syntax.WhileStatement
_WithStatement = syntax.WithStatement
_YieldExpression = syntax.YieldExpression


exports.TreeVisitor = class TreeVisitor
        # for collections, returns all non-null.  for single item properties,
        # just always returns the item.
        filter = (x) ->
                return x if not Array.isArray x
        
                rv = []
                for y in x when y?
                        if Array.isArray y
                                rv = rv.concat y
                        else
                                rv.push y
                rv

        # a rather disgusting in-place filter+flatten visitor
        visitArray: (arr) ->
                i = 0
                e = arr.length
                while i < e
                        tmp = @visit arr[i]
                        if tmp is null
                                arr.splice i, 1
                                e = arr.length
                        else if Array.isArray tmp
                                tmplen = tmp.length
                                if tmplen > 0
                                        tmp.unshift 1
                                        tmp.unshift i
                                        arr.splice.apply arr, tmp
                                        i += tmplen
                                        e = arr.length
                                else
                                        arr.splice i, 1
                                        e = arr.length
                        else
                                arr[i] = tmp
                                i += 1
                arr
                
                
        visit: (n) ->
                return null if not n?

                return @visitArray n if Array.isArray n

                #debug.indent()
                #debug.log -> "#{n.type}>"

                switch n.type
                        when _Program                 then rv = @visitProgram n
                        when _FunctionDeclaration     then rv = @visitFunctionDeclaration n
                        when _FunctionExpression      then rv = @visitFunctionExpression n
                        when _ArrowFunctionExpression then rv = @visitArrowFunctionExpression n
                        when _LabeledStatement        then rv = @visitLabeledStatement n
                        when _BlockStatement          then rv = @visitBlock n
                        when _ExpressionStatement     then rv = @visitExpressionStatement n
                        when _SwitchStatement         then rv = @visitSwitch n
                        when _SwitchCase              then rv = @visitCase n
                        when _ForStatement            then rv = @visitFor n
                        when _WhileStatement          then rv = @visitWhile n
                        when _IfStatement             then rv = @visitIf n
                        when _ForInStatement          then rv = @visitForIn n
                        when _ForOfStatement          then rv = @visitForOf n
                        when _DoWhileStatement        then rv = @visitDo n
                        when _BreakStatement          then rv = @visitBreak n
                        when _ContinueStatement       then rv = @visitContinue n
                        when _TryStatement            then rv = @visitTry n
                        when _CatchClause             then rv = @visitCatchClause n
                        when _ThrowStatement          then rv = @visitThrow n
                        when _ReturnStatement         then rv = @visitReturn n
                        when _WithStatement           then rv = @visitWith n
                        when _VariableDeclaration     then rv = @visitVariableDeclaration n
                        when _VariableDeclarator      then rv = @visitVariableDeclarator n
                        when _LabeledStatement        then rv = @visitLabeledStatement n
                        when _AssignmentExpression    then rv = @visitAssignmentExpression n
                        when _ConditionalExpression   then rv = @visitConditionalExpression n
                        when _LogicalExpression       then rv = @visitLogicalExpression n
                        when _NewExpression           then rv = @visitNewExpression n
                        when _ThisExpression          then rv = @visitThisExpression n
                        when _BinaryExpression        then rv = @visitBinaryExpression n
                        when _UnaryExpression         then rv = @visitUnaryExpression n
                        when _UpdateExpression        then rv = @visitUpdateExpression n
                        when _MemberExpression        then rv = @visitMemberExpression n
                        when _SequenceExpression      then rv = @visitSequenceExpression n
                        when _ObjectExpression        then rv = @visitObjectExpression n
                        when _ArrayExpression         then rv = @visitArrayExpression n
                        when _Identifier              then rv = @visitIdentifier n
                        when _Literal                 then rv = @visitLiteral n
                        when _CallExpression          then rv = @visitCallExpression n
                        when _Property                then rv = @visitProperty n
                        when _EmptyStatement          then rv = @visitEmptyStatement n
                        when _ClassDeclaration        then rv = @visitClassDeclaration n
                        when _ClassBody               then rv = @visitClassBody n
                        when _MethodDefinition        then rv = @visitMethodDefinition n
                        when _ExportDeclaration       then rv = @visitExportDeclaration n
                        when _ModuleDeclaration       then rv = @visitModuleDeclaration n
                        when _ImportDeclaration       then rv = @visitImportDeclaration n
                        when _ImportSpecifier         then rv = @visitImportSpecifier n
                        else
                            throw "PANIC: unknown parse node type #{n.type}"
                
                #debug.log -> "<#{n.type}, rv = #{if rv then rv.type else 'null'}"
                #debug.unindent()

                return n if rv is undefined or rv is n
                return rv
<commit_msg>throw an error, not a string, so we get a nice stacktrace from node.<commit_after>syntax = (require 'esprima').Syntax
debug = require 'debug'

_ArrayExpression = syntax.ArrayExpression
_ArrayPattern = syntax.ArrayPattern
_ArrowFunctionExpression = syntax.ArrowFunctionExpression
_AssignmentExpression = syntax.AssignmentExpression
_BinaryExpression = syntax.BinaryExpression
_BlockStatement = syntax.BlockStatement
_BreakStatement = syntax.BreakStatement
_CallExpression = syntax.CallExpression
_CatchClause = syntax.CatchClause
_ClassBody = syntax.ClassBody
_ClassDeclaration = syntax.ClassDeclaration
_ClassExpression = syntax.ClassExpression
_ClassHeritage = syntax.ClassHeritage
_ComprehensionBlock = syntax.ComprehensionBlock
_ComprehensionExpression = syntax.ComprehensionExpression
_ConditionalExpression = syntax.ConditionalExpression
_ContinueStatement = syntax.ContinueStatement
_DebuggerStatement = syntax.DebuggerStatement
_DoWhileStatement = syntax.DoWhileStatement
_EmptyStatement = syntax.EmptyStatement
_ExportDeclaration = syntax.ExportDeclaration
_ExportBatchSpecifier = syntax.ExportBatchSpecifier
_ExportSpecifier = syntax.ExportSpecifier
_ExpressionStatement = syntax.ExpressionStatement
_ForInStatement = syntax.ForInStatement
_ForOfStatement = syntax.ForOfStatement
_ForStatement = syntax.ForStatement
_FunctionDeclaration = syntax.FunctionDeclaration
_FunctionExpression = syntax.FunctionExpression
_Identifier = syntax.Identifier
_IfStatement = syntax.IfStatement
_ImportDeclaration = syntax.ImportDeclaration
_ImportSpecifier = syntax.ImportSpecifier
_LabeledStatement = syntax.LabeledStatement
_Literal = syntax.Literal
_LogicalExpression = syntax.LogicalExpression
_MemberExpression = syntax.MemberExpression
_MethodDefinition = syntax.MethodDefinition
_ModuleDeclaration = syntax.ModuleDeclaration
_NewExpression = syntax.NewExpression
_ObjectExpression = syntax.ObjectExpression
_ObjectPattern = syntax.ObjectPattern
_Program = syntax.Program
_Property = syntax.Property
_ReturnStatement = syntax.ReturnStatement
_SequenceExpression = syntax.SequenceExpression
_SpreadElement = syntax.SpreadElement
_SwitchCase = syntax.SwitchCase
_SwitchStatement = syntax.SwitchStatement
_TaggedTemplateExpression = syntax.TaggedTemplateExpression
_TemplateElement = syntax.TemplateElement
_TemplateLiteral = syntax.TemplateLiteral
_ThisExpression = syntax.ThisExpression
_ThrowStatement = syntax.ThrowStatement
_TryStatement = syntax.TryStatement
_UnaryExpression = syntax.UnaryExpression
_UpdateExpression = syntax.UpdateExpression
_VariableDeclaration = syntax.VariableDeclaration
_VariableDeclarator = syntax.VariableDeclarator
_WhileStatement = syntax.WhileStatement
_WithStatement = syntax.WithStatement
_YieldExpression = syntax.YieldExpression


exports.TreeVisitor = class TreeVisitor
        # for collections, returns all non-null.  for single item properties,
        # just always returns the item.
        filter = (x) ->
                return x if not Array.isArray x
        
                rv = []
                for y in x when y?
                        if Array.isArray y
                                rv = rv.concat y
                        else
                                rv.push y
                rv

        # a rather disgusting in-place filter+flatten visitor
        visitArray: (arr) ->
                i = 0
                e = arr.length
                while i < e
                        tmp = @visit arr[i]
                        if tmp is null
                                arr.splice i, 1
                                e = arr.length
                        else if Array.isArray tmp
                                tmplen = tmp.length
                                if tmplen > 0
                                        tmp.unshift 1
                                        tmp.unshift i
                                        arr.splice.apply arr, tmp
                                        i += tmplen
                                        e = arr.length
                                else
                                        arr.splice i, 1
                                        e = arr.length
                        else
                                arr[i] = tmp
                                i += 1
                arr
                
                
        visit: (n) ->
                return null if not n?

                return @visitArray n if Array.isArray n

                #debug.indent()
                #debug.log -> "#{n.type}>"
                
                switch n.type
                        when _Program                 then rv = @visitProgram n
                        when _FunctionDeclaration     then rv = @visitFunctionDeclaration n
                        when _FunctionExpression      then rv = @visitFunctionExpression n
                        when _ArrowFunctionExpression then rv = @visitArrowFunctionExpression n
                        when _LabeledStatement        then rv = @visitLabeledStatement n
                        when _BlockStatement          then rv = @visitBlock n
                        when _ExpressionStatement     then rv = @visitExpressionStatement n
                        when _SwitchStatement         then rv = @visitSwitch n
                        when _SwitchCase              then rv = @visitCase n
                        when _ForStatement            then rv = @visitFor n
                        when _WhileStatement          then rv = @visitWhile n
                        when _IfStatement             then rv = @visitIf n
                        when _ForInStatement          then rv = @visitForIn n
                        when _ForOfStatement          then rv = @visitForOf n
                        when _DoWhileStatement        then rv = @visitDo n
                        when _BreakStatement          then rv = @visitBreak n
                        when _ContinueStatement       then rv = @visitContinue n
                        when _TryStatement            then rv = @visitTry n
                        when _CatchClause             then rv = @visitCatchClause n
                        when _ThrowStatement          then rv = @visitThrow n
                        when _ReturnStatement         then rv = @visitReturn n
                        when _WithStatement           then rv = @visitWith n
                        when _VariableDeclaration     then rv = @visitVariableDeclaration n
                        when _VariableDeclarator      then rv = @visitVariableDeclarator n
                        when _LabeledStatement        then rv = @visitLabeledStatement n
                        when _AssignmentExpression    then rv = @visitAssignmentExpression n
                        when _ConditionalExpression   then rv = @visitConditionalExpression n
                        when _LogicalExpression       then rv = @visitLogicalExpression n
                        when _NewExpression           then rv = @visitNewExpression n
                        when _ThisExpression          then rv = @visitThisExpression n
                        when _BinaryExpression        then rv = @visitBinaryExpression n
                        when _UnaryExpression         then rv = @visitUnaryExpression n
                        when _UpdateExpression        then rv = @visitUpdateExpression n
                        when _MemberExpression        then rv = @visitMemberExpression n
                        when _SequenceExpression      then rv = @visitSequenceExpression n
                        when _ObjectExpression        then rv = @visitObjectExpression n
                        when _ArrayExpression         then rv = @visitArrayExpression n
                        when _Identifier              then rv = @visitIdentifier n
                        when _Literal                 then rv = @visitLiteral n
                        when _CallExpression          then rv = @visitCallExpression n
                        when _Property                then rv = @visitProperty n
                        when _EmptyStatement          then rv = @visitEmptyStatement n
                        when _ClassDeclaration        then rv = @visitClassDeclaration n
                        when _ClassBody               then rv = @visitClassBody n
                        when _MethodDefinition        then rv = @visitMethodDefinition n
                        when _ExportDeclaration       then rv = @visitExportDeclaration n
                        when _ModuleDeclaration       then rv = @visitModuleDeclaration n
                        when _ImportDeclaration       then rv = @visitImportDeclaration n
                        when _ImportSpecifier         then rv = @visitImportSpecifier n
                        else
                            throw new Error "PANIC: unknown parse node type #{n.type}"
                
                #debug.log -> "<#{n.type}, rv = #{if rv then rv.type else 'null'}"
                #debug.unindent()

                return n if rv is undefined or rv is n
                return rv

{'fraction_non_alphanumeric': 0.03919942500138221, 'fraction_numerical': 0.0006634599436059048, 'mean_word_length': 1.543664744761637, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2812233', 'n_tokens_mistral': 3760, 'n_tokens_neox': 3596, 'n_words': 1161}