<commit_before><?php

require_once("Tag.php");

class NoAttributeFoundException extends Exception {}
class NoTagNameFoundException extends Exception {}

class Parsing {
  //unclosed tags
  const re_allUnclosedTags = '/<[^>]+\/?>/';
  const re_allUnclosedNamedTags = '/<%s[^>]+\/?>/';
  const re_unclosedTag = '/<([^\s]+)(.+)\/?>/';
  const re_unclosedNamedTag = '/<(%s)(.+)\/?>/';

  //closed tags
  const re_closedNamedTags = '/<%s([^>]*)(?!\/)>(.*?)<\/%s>/s';

  //attributes
  const re_attributePair = '/\s*([^=\s]+)\s*=\s*"([^"]+)"/';

  public static function parseAllUnclosedTags($str, $name=null) {
    $re = !is_string($name) ? self::re_allUnclosedTags : sprintf(self::re_allUnclosedNamedTags, $name);

    $ret = preg_match_all($re, $str, $matches);
    if($ret===false || $ret<=0) throw new NoTagNameFoundException("unable to find <".(empty($name) ? 'tag' : $name)."> elements");

    $tags = array();
    foreach($matches[0] as $tag_match) {
      array_push($tags, self::parseUnclosedTag($tag_match, $name));
    }
    return $tags;
  }

  public static function parseUnclosedTag($str, $name=null) {
    $re = !is_string($name) ? re_unclosedTag : sprintf(self::re_unclosedNamedTag, $name);

    $ret = preg_match($re, $str, $matches);
    if($ret===false || $ret<=0) throw new NoTagNameFoundException("unable to find <".(empty($name) ? 'tag' : $name)."> elements");
    $tag_name = $matches[1];

    if(array_key_exists(2, $matches)) {
      try {
        $tag_attributes = self::parseAttributes($matches[2]);
      } catch (NoAttributeFoundException $e) {
        //no attributes found
        $tag_attributes = null;
      }
    } else {
      $tag_attributes = null;
    }

    $tag = new Tag($tag_name, $tag_attributes);
    return $tag;
  }

  public static function parseAllClosedNamedTags($str, $name) {
    $re = sprintf(self::re_closedNamedTags, $name, $name);

    $ret = preg_match_all($re, $str, $matches);
    if($ret===false || $ret<=0) throw new NoTagNameFoundException("unable to find <".$name."> elements");

    $tag_count = count($matches[0]);
    $tags = array();
    for($i=0;$i<$tag_count;$i++) {
      try {
        $tag_attributes = self::parseAttributes($matches[1][$i]);
      } catch (NoAttributeFoundException $e) {
        //no attributes found
        $tag_attributes = array();
      }
      $tag = new Tag($name, $tag_attributes, $matches[2][$i]);
      array_push($tags, $tag);
    }

    return $tags;
  }

  public static function parseAttributes($attr_str) {
    $ret = preg_match_all(self::re_attributePair, $attr_str, $matches);
    if($ret===false || $ret<=0) throw new NoAttributeFoundException("unable to find attributes");

    $attrs = array_combine($matches[1], $matches[2]);
    return $attrs;
  }
}

?><commit_msg>update regexps; don't need one for general, one for specific<commit_after><?php

require_once("Tag.php");

class NoAttributeFoundException extends Exception {}
class NoTagNameFoundException extends Exception {}

class Parsing {
  //unclosed tags
  const re_unclosedTag = '/<([^\s\/>]+)(?:(\s+[^>]*?)\/?>|\/?>)/';
  const re_unclosedNamedTag = '/<(%s)(?:(\s+[^>]*?)\/?>|\/?>)/';

  //closed tags
  const re_closedNamedTags = '/<%s([^>]*)(?!\/)>(.*?)<\/%s>/s';

  //attributes
  const re_attributePair = '/\s*([^=\s]+)\s*=\s*"([^"]+)"/';

  public static function parseAllUnclosedTags($str, $name=null) {
    $re = !is_string($name) ? self::re_unclosedTag : sprintf(self::re_unclosedNamedTag, $name);

    $ret = preg_match_all($re, $str, $matches);
    if($ret===false || $ret<=0) throw new NoTagNameFoundException("unable to find <".(empty($name) ? 'tag' : $name)."> elements");

    $tags = array();
    foreach($matches[0] as $tag_match) {
      array_push($tags, self::parseUnclosedTag($tag_match, $name));
    }
    return $tags;
  }

  public static function parseUnclosedTag($str, $name=null) {
    $re = !is_string($name) ? re_unclosedTag : sprintf(self::re_unclosedNamedTag, $name);

    $ret = preg_match($re, $str, $matches);
    if($ret===false || $ret<=0) throw new NoTagNameFoundException("unable to find <".(empty($name) ? 'tag' : $name)."> elements");
    $tag_name = $matches[1];

    if(array_key_exists(2, $matches)) {
      try {
        $tag_attributes = self::parseAttributes($matches[2]);
      } catch (NoAttributeFoundException $e) {
        //no attributes found
        $tag_attributes = null;
      }
    } else {
      $tag_attributes = null;
    }

    $tag = new Tag($tag_name, $tag_attributes);
    return $tag;
  }

  public static function parseAllClosedNamedTags($str, $name) {
    $re = sprintf(self::re_closedNamedTags, $name, $name);

    $ret = preg_match_all($re, $str, $matches);
    if($ret===false || $ret<=0) throw new NoTagNameFoundException("unable to find <".$name."> elements");

    $tag_count = count($matches[0]);
    $tags = array();
    for($i=0;$i<$tag_count;$i++) {
      try {
        $tag_attributes = self::parseAttributes($matches[1][$i]);
      } catch (NoAttributeFoundException $e) {
        //no attributes found
        $tag_attributes = array();
      }
      $tag = new Tag($name, $tag_attributes, $matches[2][$i]);
      array_push($tags, $tag);
    }

    return $tags;
  }

  public static function parseAttributes($attr_str) {
    $ret = preg_match_all(self::re_attributePair, $attr_str, $matches);
    if($ret===false || $ret<=0) throw new NoAttributeFoundException("unable to find attributes");

    $attrs = array_combine($matches[1], $matches[2]);
    return $attrs;
  }
}

?>
{'fraction_non_alphanumeric': 0.19170620277631348, 'fraction_numerical': 0.004920049200492005, 'mean_word_length': 3.3318112633181127, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3776161', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1873, 'n_words': 376}