<commit_before><?php
namespace Raideer\Tweech\Util;

class Parser{

  protected $messageRegex;
  protected $paramsRegex;

  public function __construct(){

    /**
     * Regex for parsing the message
     * http://tools.ietf.org/html/rfc1459#section-2.3.1
     */

    $space = " ";
    $null = "\\x00";
    $crlf = "\r\n";
    $letters = "A-Za-z";
    $numbers = "0-9";
    $special = preg_quote('[]\`_^{|}');

    $trailing = "[^$null$crlf]*";
    $username = "[$letters$numbers$special]+";
    $server = "(?:(?:[$letters$numbers\.]*)\.(?:[$letters$numbers]+)\.(?:[$letters]+))";

    $prefixFull = "((?P<username>$username)(!$username)?(@$username)?\.(?P<server>$server))";
    $prefixPart = "(?:(?P<usernamep>$username)\.(?P<serverp>$server))";
    $prefixSmall = "(?:(?P<servers>$server))";

    $command = "(?P<command>[$letters]+|[$numbers]{3})";

    $params = "(?P<params>$trailing)";

    $prefix = "(?:$prefixFull|$prefixPart|$prefixSmall)";
    $compiled = "(?P<prefix>:$prefix)?[$space]$command$space$params$crlf";

    $this->messageRegex = "/^$compiled$/U";

    $this->paramsRegex = array(
      'PRIVMSG' => "/^(?P<channel>#$username)[$space]?:(?P<message>$trailing)$/s",
      '372' => "/^(?P<username>$username)[$space]?:(?P<motd>$trailing)$/s",
      '001' => "/^(?P<username>$username)[$space]?:(?P<welcome>$trailing)$/s",
      '002' => "/^(?P<username>$username)[$space]?:(?P<host>$trailing)$/s",
      '033' => "/^(?P<username>$username)[$space]?:(?P<created>$trailing)$/s"
    );
  }

  protected function removeIntegerKeys(array $array){
    foreach (array_keys($array) as $key) {
        if (is_int($key)) {
            unset($array[$key]);
        }
    }
    return $array;
  }

  protected function copyAndDelete($from,$to,$parsed){
    if(array_key_exists($from,$parsed)){
      if($parsed[$to] != null){
        unset($parsed[$from]);
        return $parsed;
      }
      $parsed[$to] = $parsed[$from];
      unset($parsed[$from]);
    }

    return $parsed;
  }

  protected function parseParameters($parsed){
    $command = strtoupper($parsed['command']);
    if(!array_key_exists($command, $this->paramsRegex)) return $parsed;


    if(!preg_match($this->paramsRegex[$command], $parsed['params'], $params)) return $parsed;

    $params = $this->removeIntegerKeys($params);
    $parsed = array_merge($parsed, $params);

    return array_unique($parsed);
  }

  public function parse($message){
    if(strpos($message, "\r\n") === false){<commit_msg>Fixed when "full" key was not appearing<commit_after><?php
namespace Raideer\Tweech\Util;

class Parser{

  protected $messageRegex;
  protected $paramsRegex;

  public function __construct(){

    /**
     * Regex for parsing the message
     * http://tools.ietf.org/html/rfc1459#section-2.3.1
     */

    $space = " ";
    $null = "\\x00";
    $crlf = "\r\n";
    $letters = "A-Za-z";
    $numbers = "0-9";
    $special = preg_quote('[]\`_^{|}');

    $trailing = "[^$null$crlf]*";
    $username = "[$letters$numbers$special]+";
    $server = "(?:(?:[$letters$numbers\.]*)\.(?:[$letters$numbers]+)\.(?:[$letters]+))";

    $prefixFull = "((?P<username>$username)(!$username)?(@$username)?\.(?P<server>$server))";
    $prefixPart = "(?:(?P<usernamep>$username)\.(?P<serverp>$server))";
    $prefixSmall = "(?:(?P<servers>$server))";

    $command = "(?P<command>[$letters]+|[$numbers]{3})";

    $params = "(?P<params>$trailing)";

    $prefix = "(?:$prefixFull|$prefixPart|$prefixSmall)";
    $compiled = "(?P<prefix>:$prefix)?[$space]$command$space$params$crlf";

    $this->messageRegex = "/^$compiled$/U";

    $this->paramsRegex = array(
      'PRIVMSG' => "/^(?P<channel>#$username)[$space]?:(?P<message>$trailing)$/s",
      '372' => "/^(?P<username>$username)[$space]?:(?P<motd>$trailing)$/s",
      '001' => "/^(?P<username>$username)[$space]?:(?P<welcome>$trailing)$/s",
      '002' => "/^(?P<username>$username)[$space]?:(?P<host>$trailing)$/s",
      '033' => "/^(?P<username>$username)[$space]?:(?P<created>$trailing)$/s"
    );
  }

  protected function removeIntegerKeys(array $array){
    foreach (array_keys($array) as $key) {
        if (is_int($key)) {
            unset($array[$key]);
        }
    }
    return $array;
  }

  protected function copyAndDelete($from,$to,$parsed){
    if(array_key_exists($from,$parsed)){
      if($parsed[$to] != null){
        unset($parsed[$from]);
        return $parsed;
      }
      $parsed[$to] = $parsed[$from];
      unset($parsed[$from]);
    }

    return $parsed;
  }

  protected function parseParameters($parsed){
    $command = strtoupper($parsed['command']);
    if(!array_key_exists($command, $this->paramsRegex)) return $parsed;


    if(!preg_match($this->paramsRegex[$command], $parsed['params'], $params)) return $parsed;

    $parsed = array_merge($parsed, $params);

    return $this->removeIntegerKeys($parsed);
  }

  public function parse($message){
    if(strpos($message, "\r\n") === false){
{'fraction_non_alphanumeric': 0.24521072796934865, 'fraction_numerical': 0.009679370840895343, 'mean_word_length': 3.96, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6407748', 'n_tokens_mistral': 1846, 'n_tokens_neox': 1735, 'n_words': 237}