<commit_before><?php 

	namespace HippoPHP\Tokenizer;

	class Token {
		public function __construct() {
			return $this;
		}
	}<commit_msg>Taken Token.php from Hippo, without the namespace code<commit_after><?php

	namespace HippoPHP\Tokenizer;

	class Token {
		/**
		 * Token type.
		 * @var mixed
		 */
		protected $type;

		/**
		 * Token lexme.
		 * @var int
		 */
		protected $lexeme;

		/**
		 * Token line.
		 * @var int
		 */
		protected $line;

		/**
		 * Token column.
		 * @var int
		 */
		protected $column;

		/**
		 * Token nest level.
		 * @var int
		 */
		protected $level;

		/**
		 * Creates a new Token.
		 * @param mixed $type
		 * @param string $lexeme
		 * @param int $line
		 * @param int $column
		 * @return Token
		 */
		public function __construct($type, $lexeme, $line, $column) {
			$this->type = $type;
			$this->lexeme = $lexeme;
			$this->line = $line;
			$this->column = $column;

			return $this;
		}

		/**
		 * Return the token type.
		 * @return mixed
		 */
		public function getType() {
			return $this->type;
		}

		/**
		 * Return the token lexme.
		 * @return string
		 */
		public function getLexeme() {
			return $this->lexeme;
		}

		/**
		 * Return the token line.
		 * @return int
		 */
		public function getLine() {
			return $this->line;
		}

		/**
		 * Return the token column.
		 * @return int
		 */
		public function getColumn() {
			return $this->column;
		}

		/**
		 * Set the nested level of the token.
		 * @param int $level
		 * @return Token
		 */
		public function setLevel($level) {
			$this->level = $level;
			return $this;
		}

		/**
		 * Get the nested level of the token.
		 * @return int
		 */
		public function getLevel() {
			return $this->level;
		}

		/**
		 * Check if the token contains new line characters.
		 * @return boolean
		 */
		public function hasNewline() {
			if (preg_match('([\r\n])', $this->lexeme)) {
				return true;
			}

			return false;
		}

		/**
		 * Get the number of new lines.
		 * @return int
		 */
		public function getNewlineCount() {
			preg_match_all('(\n|\r\n?)', $this->lexeme, $matches, PREG_SET_ORDER);
			return count($matches);
		}

		/**
		 * Get the length of the last line.
		 * @return int
		 */
		public function getTrailingLineLength() {
			return iconv_strlen(
				substr(strrchr($this->lexeme, "\n") ?: strrchr($this->lexeme, "\r"), 1),
				'utf-8'
			);
		}

		/**
		 * Get length of entire token lexme.
		 * @return int
		 */
		public function getLength() {
			return iconv_strlen($this->lexeme, 'utf-8');
		}
	}
{'fraction_non_alphanumeric': 0.15177865612648223, 'fraction_numerical': 0.0011857707509881424, 'mean_word_length': 2.2998696219035204, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7505935', 'n_tokens_mistral': 1057, 'n_tokens_neox': 827, 'n_words': 235}