<commit_before><?php
/**
 * Yarrow {{version}}
 * Simple Documentation Generator
 * <http://yarrowdoc.org>
 *
 * Copyright (c) 2010-2011, <NAME> <http://maetl.net>
 * All rights reserved.
 * 
 * This library is free software; refer to the terms in the LICENSE file found
 * with this source code for details about modification and redistribution.
 */

define('T_SCOPE_START', 900);
define('T_SCOPE_END', 909);
define('T_BRACE_OPEN', 911);
define('T_BRACE_CLOSE', 919);
define('T_TERNARY_IF', 977);
define('T_UNKNOWN', 999);

/**
 * Parse a PHP token stream and extract useful information about its
 * object oriented structure.
 */
class CodeParser {
	private $tokens;
	private $total;
	private $current;
	private $line;
	private $scope;
	private $file;
	private $class;
	private $function;
	private $tokenMap = array(
		'{' => T_SCOPE_START,
		'}' => T_SCOPE_END,
		'(' => T_BRACE_OPEN,
		')' => T_BRACE_CLOSE,
		'?' => T_TERNARY_IF
	);
	
	const GLOBAL_SCOPE 	 = 1;
	const CLASS_SCOPE  	 = 2;
	const FUNCTION_SCOPE = 3;
	
	/**
	 * @param $tokens array of token identifiers
	 * @param $reader CodeReader instance
	 */
	function __construct($tokens, $reader) {
		$line = 1;
		
		foreach($tokens as $i => $token) {
			if (!is_array($token)) {
				$tokens[$i] = $this->normalizeToken($token, $line);
			}
			$lines = substr_count($tokens[$i][1], "\n");
			$line += $lines;
		}
		
		$this->tokens = $tokens;
		$this->total = count($this->tokens);
		$this->current = 0;
		$this->state = self::GLOBAL_SCOPE;
		$this->scope = new ScopeStack();
		$this->reader = $reader;
		$this->keywords = array();
		
		//----------------------------------------------------------------

		$this->docblockScope = false; // replace these with a stack
		$this->classScope = false;	  // that holds nested scope state
	}
	
	/**
	 * Converts strings from the PHP tokenizer to the standard format.
	 */
	function normalizeToken($token, $line) {
		$symbol = (isset($this->tokenMap[$token])) ? $this->tokenMap[$token] : T_UNKNOWN;
		return array($symbol, $token, $line);
	}
	
	/**
	 * Shreds a PHP code file for classes, methods, and doc comments.
	 */
	function parse() {
		
		for ($this->current = 0; $this->current < $this->total; $this->current++) {
			
			$token = $this->currentToken();
						
			$this->symbol = $token[0];
			$this->value = $token[1];
			$this->line = $token[1];
			
			switch ($this->symbol) {
				
				case T_OPEN_TAG:
					$this->scope->push($this->state);
				break;
				
				case T_DOC_COMMENT:
					$this->shredDocBlock();
				break;

				case T_CLASS:<commit_msg>adding detection of interface token - #3<commit_after><?php
/**
 * Yarrow {{version}}
 * Simple Documentation Generator
 * <http://yarrowdoc.org>
 *
 * Copyright (c) 2010-2011, <NAME> <http://maetl.net>
 * All rights reserved.
 * 
 * This library is free software; refer to the terms in the LICENSE file found
 * with this source code for details about modification and redistribution.
 */

define('T_SCOPE_START', 900);
define('T_SCOPE_END', 909);
define('T_BRACE_OPEN', 911);
define('T_BRACE_CLOSE', 919);
define('T_TERNARY_IF', 977);
define('T_UNKNOWN', 999);

/**
 * Parse a PHP token stream and extract useful information about its
 * object oriented structure.
 */
class CodeParser {
	private $tokens;
	private $total;
	private $current;
	private $line;
	private $scope;
	private $file;
	private $class;
	private $function;
	private $tokenMap = array(
		'{' => T_SCOPE_START,
		'}' => T_SCOPE_END,
		'(' => T_BRACE_OPEN,
		')' => T_BRACE_CLOSE,
		'?' => T_TERNARY_IF
	);
	
	const GLOBAL_SCOPE 	 = 1;
	const CLASS_SCOPE  	 = 2;
	const FUNCTION_SCOPE = 3;
	
	/**
	 * @param $tokens array of token identifiers
	 * @param $reader CodeReader instance
	 */
	function __construct($tokens, $reader) {
		$line = 1;
		
		foreach($tokens as $i => $token) {
			if (!is_array($token)) {
				$tokens[$i] = $this->normalizeToken($token, $line);
			}
			$lines = substr_count($tokens[$i][1], "\n");
			$line += $lines;
		}
		
		$this->tokens = $tokens;
		$this->total = count($this->tokens);
		$this->current = 0;
		$this->state = self::GLOBAL_SCOPE;
		$this->scope = new ScopeStack();
		$this->reader = $reader;
		$this->keywords = array();
		
		//----------------------------------------------------------------

		$this->docblockScope = false; // replace these with a stack
		$this->classScope = false;	  // that holds nested scope state
	}
	
	/**
	 * Converts strings from the PHP tokenizer to the standard format.
	 */
	function normalizeToken($token, $line) {
		$symbol = (isset($this->tokenMap[$token])) ? $this->tokenMap[$token] : T_UNKNOWN;
		return array($symbol, $token, $line);
	}
	
	/**
	 * Shreds a PHP code file for classes, methods, and doc comments.
	 */
	function parse() {
		
		for ($this->current = 0; $this->current < $this->total; $this->current++) {
			
			$token = $this->currentToken();
						
			$this->symbol = $token[0];
			$this->value = $token[1];
			$this->line = $token[1];
			
			switch ($this->symbol) {
				
				case T_OPEN_TAG:
					$this->scope->push($this->state);
				break;
				
				case T_DOC_COMMENT:
					$this->shredDocBlock();
				break;

				case T_CLASS:
				case T_INTERFACE:
{'fraction_non_alphanumeric': 0.1864013907668534, 'fraction_numerical': 0.014100830596870775, 'mean_word_length': 3.1160572337042924, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2809180', 'n_tokens_mistral': 2029, 'n_tokens_neox': 1717, 'n_words': 442}