<commit_before><?php

class CRM_Speakcivi_Tools_Helper {

  /**
   * Clean out string from unicode special chars.
   * Those unicode chars invoke bugs during insert/update on db table.
   *
   * @param string $string
   *
   * @return mixed
   */
  public static function cleanUnicodeChars($string) {
    $forbiddenRegex = '/['
      . '\x00-\x1F\x80-\xFF'
      . '\x{1F600}-\x{1F64F}'  // Match Emoticons
      . '\x{1F300}-\x{1F5FF}'  // Match Miscellaneous Symbols and Pictographs
      . '\x{1F680}-\x{1F6FF}'  // Match Transport And Map Symbols
      . '\x{2600}-\x{26FF}'  // Match Miscellaneous Symbols
      . '\x{2700}-\x{27BF}'  // Match Dingbats
      . ']/u';
    return preg_replace($forbiddenRegex, '', $string);
  }


  /**
   * Trim all strings from object.
   *
   * @param object $param
   */
  public static function trimVariables(&$param) {
    foreach ($param as $key => $value) {
      if (is_object($value)) {
        self::trimVariables($value);
      } elseif (is_array($value)) {
        foreach($value as $k => $v) {
          self::trimVariables($v);
        }
      } elseif (is_string($value)) {
        $param->$key = self::clean($value, $key);
      }
    }
  }


  /**
   * Clean out string from redundant spaces and trim to valid length.
   *
   * @param string $value
   * @param string $type
   *
   * @return string
   */
  private static function clean($value, $type = '') {
    $value = self::cleanUnicodeChars($value);
    $value = self::cleanSpaces($value);
    switch ($type) {
      case 'phone':
        $value = mb_substr($value, 0, 32);
        break;
      case 'firstname':
      case 'lastname':
      case 'zip':
        $value = mb_substr($value, 0, 64);
        break;
      case 'email':
        $value = mb_substr($value, 0, 254);
        break;
    }
    return $value;
  }


  /**
   * Reduce redundant spaces.
   *
   * @param string $value
   *
   * @return string
   */
  private static function cleanSpaces($value) {
    return preg_replace('/\s+/', ' ', trim($value));
  }
}
<commit_msg>erge branch 'master' of https://github.com/WeMoveEU/speakcivi<commit_after><?php

class CRM_Speakcivi_Tools_Helper {

  /**
   * Clean out string from unicode special chars not supported by MySQL 5.7.
   * "No support for supplementary characters (BMP characters only)."
   *
   * @link https://dev.mysql.com/doc/refman/5.7/en/charset-unicode-utf8.html Description of utf8 in MySQL documentation
   * @link http://www.fileformat.info/info/unicode/block/index.htm List of blocks for checking
   *
   * @param string $string
   *
   * @return mixed
   */
  public static function cleanUnicodeChars($string) {
    $forbiddenRegex = '/['
      . '\x{10000}-\x{1F9FF}'  // The second level (U+10000 to U+1FFFF) is the Supplementary Multilingual Plane (Plane 1, SMP)
      . '\x{20000}-\x{2FFFF}'  // The second level (U+20000 to U+2FFFF) is the Supplementary Ideographic Plane (Plane 2, SIP)
      . '\x{2600}-\x{26FF}'  // Match Miscellaneous Symbols
      . '\x{2700}-\x{27BF}'  // Match Dingbats
      . ']/u';
    return preg_replace($forbiddenRegex, '', $string);
  }


  /**
   * Trim all strings from object.
   *
   * @param object $param
   */
  public static function trimVariables(&$param) {
    foreach ($param as $key => $value) {
      if (is_object($value)) {
        self::trimVariables($value);
      } elseif (is_array($value)) {
        foreach($value as $k => $v) {
          self::trimVariables($v);
        }
      } elseif (is_string($value)) {
        $param->$key = self::clean($value, $key);
      }
    }
  }


  /**
   * Clean out string from redundant spaces and trim to valid length.
   *
   * @param string $value
   * @param string $type
   *
   * @return string
   */
  private static function clean($value, $type = '') {
    $value = self::cleanUnicodeChars($value);
    $value = self::cleanSpaces($value);
    switch ($type) {
      case 'phone':
        $value = mb_substr($value, 0, 32);
        break;
      case 'firstname':
      case 'lastname':
      case 'zip':
        $value = mb_substr($value, 0, 64);
        break;
      case 'email':
        $value = mb_substr($value, 0, 254);
        break;
    }
    return $value;
  }


  /**
   * Reduce redundant spaces.
   *
   * @param string $value
   *
   * @return string
   */
  private static function cleanSpaces($value) {
    return preg_replace('/\s+/', ' ', trim($value));
  }
}

{'fraction_non_alphanumeric': 0.1555857078245138, 'fraction_numerical': 0.022840343735866124, 'mean_word_length': 2.378915202444614, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4202525', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1448, 'n_words': 358}