private MemoEntry processTokenDefinition ( ParseTreeNode node , TokenDefinition tokenDefinition , int position , int line ) throws TreeException { Matcher matcher = tokenDefinition . getPattern ( ) . matcher ( text . substring ( position ) ) ; if ( ! matcher . find ( ) ) { return MemoEntry . failed ( ) ; } String match = matcher . group ( ) ; int lineBreakNum = StringUtils . countLineBreaks ( match ) ; SourceCodeLocation source = sourceCode . getLines ( ) . get ( line - 1 ) . getSource ( ) ; int lineNumber = textWithSource . getLineNumber ( position ) ; TokenMetaData metaData = new TokenMetaData ( source , lineNumber , lineBreakNum + 1 , textWithSource . getColumn ( position ) ) ; Token token = new Token ( tokenDefinition . getName ( ) , match , tokenDefinition . getVisibility ( ) , metaData ) ; ParseTreeNode myTree = new ParseTreeNode ( token ) ; node . addChild ( myTree ) ; if ( maxPosition < position + match . length ( ) ) { maxPosition = position + match . length ( ) ; } return MemoEntry . success ( match . length ( ) , lineBreakNum , myTree ) ; }
This method tries to process a single token definition . If this can be done true is returned a new parser tree child is added and all internal states are updated like position id and line .