<commit_before><?php
namespace Neolao\Util;

/**
 * Class utility for string manipulation
 */
class String
{
    public static $accents = array(
        'Š'=>'S', 'š'=>'s', 'Ð'=>'D', 'Ž'=>'Z', 'ž'=>'z', 'À'=>'A', 'Á'=>'A', 'Â'=>'A', 'Ã'=>'A', 'Ä'=>'A', 
        'Å'=>'A', 'Æ'=>'A', 'Ç'=>'C', 'È'=>'E', 'É'=>'E', 'Ê'=>'E', 'Ë'=>'E', 'Ì'=>'I', 'Í'=>'I', 'Î'=>'I', 
        'Ï'=>'I', 'Ñ'=>'N', 'Ò'=>'O', 'Ó'=>'O', 'Ô'=>'O', 'Õ'=>'O', 'Ö'=>'O', 'Ø'=>'O', 'Ù'=>'U', 'Ú'=>'U', 
        'Û'=>'U', 'Ü'=>'U', 'Ý'=>'Y', 'Þ'=>'B', 'ß'=>'Ss','à'=>'a', 'á'=>'a', 'â'=>'a', 'ã'=>'a', 'ä'=>'a', 
        'å'=>'a', 'æ'=>'a', 'ç'=>'c', 'è'=>'e', 'é'=>'e', 'ê'=>'e', 'ë'=>'e', 'ì'=>'i', 'í'=>'i', 'î'=>'i', 
        'ï'=>'i', 'ð'=>'o', 'ñ'=>'n', 'ò'=>'o', 'ó'=>'o', 'ô'=>'o', 'õ'=>'o', 'ö'=>'o', 'ø'=>'o', 'ù'=>'u', 
        'ú'=>'u', 'û'=>'u', 'ý'=>'y', 'ý'=>'y', 'þ'=>'b', 'ÿ'=>'y', 'ƒ'=>'f'
    );



    /**
     * Convert all applicable characters to HTML entities
     * 
     * @param   string  $text       The text to convert
     * @return  string              The converted text
     */
    public static function escapeHtml($text)
    {
        return htmlentities($text, ENT_QUOTES, 'UTF-8');
    }

    /**
     * Normalize a file name
     * 
     * @param   string  $name       The name
     * @return  string              The normalized name
     */
    public static function normalizeFileName($name)
    {
        $fileName = $name;

        //$fileName = str_replace('&', '-and-', $fileName);
        $fileName = strtr($name, self::$accents);
        $fileName = preg_replace('/[^a-z0-9_ -]/si', '', $fileName);
        $fileName = trim($fileName);
        $fileName = str_replace(' ', '-', $fileName);
        $fileName = preg_replace('/-+/', '-', $fileName);
        //$fileName = strtolower($fileName);

        return $fileName;
    }

    /**
     * Remove accents
     * 
     * @param   string  $text       The text
     * @return  string              The modified text
     */
    public static function removeAccents($text)
    {
        return strtr($text, self::$accents);
    }
}<commit_msg>Add a function to get the language from a locale string<commit_after><?php
namespace Neolao\Util;

/**
 * Class utility for string manipulation
 */
class String
{
    /**
     * Accents
     *
     * @var array
     */
    public static $accents = array(
        'Š'=>'S', 'š'=>'s', 'Ð'=>'D', 'Ž'=>'Z', 'ž'=>'z', 'À'=>'A', 'Á'=>'A', 'Â'=>'A', 'Ã'=>'A', 'Ä'=>'A', 
        'Å'=>'A', 'Æ'=>'A', 'Ç'=>'C', 'È'=>'E', 'É'=>'E', 'Ê'=>'E', 'Ë'=>'E', 'Ì'=>'I', 'Í'=>'I', 'Î'=>'I', 
        'Ï'=>'I', 'Ñ'=>'N', 'Ò'=>'O', 'Ó'=>'O', 'Ô'=>'O', 'Õ'=>'O', 'Ö'=>'O', 'Ø'=>'O', 'Ù'=>'U', 'Ú'=>'U', 
        'Û'=>'U', 'Ü'=>'U', 'Ý'=>'Y', 'Þ'=>'B', 'ß'=>'Ss','à'=>'a', 'á'=>'a', 'â'=>'a', 'ã'=>'a', 'ä'=>'a', 
        'å'=>'a', 'æ'=>'a', 'ç'=>'c', 'è'=>'e', 'é'=>'e', 'ê'=>'e', 'ë'=>'e', 'ì'=>'i', 'í'=>'i', 'î'=>'i', 
        'ï'=>'i', 'ð'=>'o', 'ñ'=>'n', 'ò'=>'o', 'ó'=>'o', 'ô'=>'o', 'õ'=>'o', 'ö'=>'o', 'ø'=>'o', 'ù'=>'u', 
        'ú'=>'u', 'û'=>'u', 'ý'=>'y', 'ý'=>'y', 'þ'=>'b', 'ÿ'=>'y', 'ƒ'=>'f'
    );



    /**
     * Convert all applicable characters to HTML entities
     * 
     * @param   string  $text       The text to convert
     * @return  string              The converted text
     */
    public static function escapeHtml($text)
    {
        return htmlentities($text, ENT_QUOTES, 'UTF-8');
    }

    /**
     * Normalize a file name
     * 
     * @param   string  $name       The name
     * @return  string              The normalized name
     */
    public static function normalizeFileName($name)
    {
        $fileName = $name;

        //$fileName = str_replace('&', '-and-', $fileName);
        $fileName = strtr($name, self::$accents);
        $fileName = preg_replace('/[^a-z0-9_ -]/si', '', $fileName);
        $fileName = trim($fileName);
        $fileName = str_replace(' ', '-', $fileName);
        $fileName = preg_replace('/-+/', '-', $fileName);
        //$fileName = strtolower($fileName);

        return $fileName;
    }

    /**
     * Remove accents
     * 
     * @param   string  $text       The text
     * @return  string              The modified text
     */
    public static function removeAccents($text)
    {
        return strtr($text, self::$accents);
    }

    /**
     * Get the language from a locale string
     *
     * @param   string      $localeString       Locale string
     * @return  string                          Language string
     */
    public static function getLanguage($localeString)
    {
        $array = explode('_', $localeString);

        return array_shift($array);
    }
}
{'fraction_non_alphanumeric': 0.289247311827957, 'fraction_numerical': 0.0012903225806451613, 'mean_word_length': 2.372733865119652, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 137, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3372042', 'n_tokens_mistral': 1619, 'n_tokens_neox': 1788, 'n_words': 376}