<commit_before><?php

class LuminousUtils {
  
  static function escape_string($string) {
  }
  static function escape_token($token) {
    if (!$token[2]) {
      $token[1] = htmlspecialchars($token[1], ENT_NOQUOTES);
      $token[2] = true;
    }
    return $token;
  }
}<commit_msg>add block-tagger and pcre err decoder to utils class<commit_after><?php
// Essentially a namespace
class LuminousUtils {

  static function escape_string($string) {
    return htmlspecialchars($string, ENT_NOQUOTES);
  }
  
  static function escape_token($token) {
    if (!$token[2]) {
      $token[1] = htmlspecialchars($token[1], ENT_NOQUOTES);
      $token[2] = true;
    }
    return $token;
  }

  static function tag_block($type, $block, $split_multiline=true) {
    if ($type === null) return $block;
    $open = '<' . $type . '>';
    $close = '</' . $type . '>';
    if ($split_multiline)
      return $open . str_replace("\n", $close . "\n" . $open, $block) .
          $close;
    else
      return $open . $block . $close;
  }

  static function pcre_error_decode($errcode) {
    switch ($errcode) {
      case PREG_NO_ERROR:
        return 'PREG_NO_ERROR';
      case PREG_INTERNAL_ERROR:
        return 'PREG_INTERNAL_ERROR';
      case PREG_BACKTRACK_LIMIT_ERROR:
        return 'PREG_BACKTRACK_LIMIT_ERROR';
      case PREG_RECURSION_LIMIT_ERROR:
        return 'PREG_RECURSION_LIMIT_ERROR';
      case PREG_BAD_UTF8_ERROR:
        return 'PREG_BAD_UTF8_ERROR';
      case PREG_BAD_UTF8_OFFSET_ERROR:
        return 'PREG_BAD_UTF8_OFFSET_ERROR';
      default:
        return "Unknown error code `$errcode'";
    }
  }
}
{'fraction_non_alphanumeric': 0.12769895126465144, 'fraction_numerical': 0.007402837754472548, 'mean_word_length': 2.9852579852579852, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6765769', 'n_tokens_mistral': 567, 'n_tokens_neox': 567, 'n_words': 114}