<commit_before><?php
/*
Some usefull functions to deal with internationalisation

Copyright 2013 <NAME> <<EMAIL>>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/


// Security check
if (!defined("WIKINI_VERSION")) { 
    die ("acc&egrave;s direct interdit"); 
}

// Compatibility with php versions without json functions
if (!function_exists('json_decode')) {
    function json_decode($content, $assoc=false) {
        require_once 'includes/JSON.php';
        if ($assoc) {
            $json = new Services_JSON(SERVICES_JSON_LOOSE_TYPE);
        }
        else {
            $json = new Services_JSON;
        }
        return $json->decode($content);
    }
}
if (!function_exists('json_encode')) {
    function json_encode($content) {
        require_once 'includes/JSON.php';
        $json = new Services_JSON;
        return $json->encode($content);
    }
}

/**
 * Translate the text in the page's language
 *
 * @param string array key for the text or false if doesn't exists
 * @return string the translated text or the key if not found
 */
function _t($textkey)
{
    if (isset($GLOBALS['translations'][$textkey])) {
        return $GLOBALS['translations'][$textkey];
    }
    else {
        return $textkey;
    }
}

/**
 * Convert the text in the page's charset, or in the datadase's charset
 *
 * @param string the text
 * @param string the page's encoding
 * @param bool is it for the database ?
 * @return string the encoded text
 */
function _convert($text, $fromencoding, $database = FALSE)
{
    if ($database) { 
        if ($fromencoding != "ISO-8859-1" && $fromencoding != "ISO-8859-15") {
            return iconv($fromencoding, "ISO-8859-15", $text);
        } 
        else {
            return $text;
        }
    }
    else {
        if ($fromencoding != TEMPLATES_DEFAULT_CHARSET) {
            return iconv($fromencoding, TEMPLATES_DEFAULT_CHARSET, $text);
        } 
        else {
            return $text;
        }<commit_msg>conversion de iso a utf8 prennant en compte les caracteres speciaux<commit_after><?php
/*
Some usefull functions to deal with internationalisation

Copyright 2013 <NAME> <<EMAIL>>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/


// Security check
if (!defined("WIKINI_VERSION")) { 
    die ("acc&egrave;s direct interdit"); 
}

// Compatibility with php versions without json functions
if (!function_exists('json_decode')) {
    function json_decode($content, $assoc=false) {
        require_once 'includes/JSON.php';
        if ($assoc) {
            $json = new Services_JSON(SERVICES_JSON_LOOSE_TYPE);
        }
        else {
            $json = new Services_JSON;
        }
        return $json->decode($content);
    }
}
if (!function_exists('json_encode')) {
    function json_encode($content) {
        require_once 'includes/JSON.php';
        $json = new Services_JSON;
        return $json->encode($content);
    }
}

/**
 * Translate the text in the page's language
 *
 * @param string array key for the text or false if doesn't exists
 * @return string the translated text or the key if not found
 */
function _t($textkey)
{
    if (isset($GLOBALS['translations'][$textkey])) {
        return $GLOBALS['translations'][$textkey];
    }
    else {
        return $textkey;
    }
}

/**
 * Convert the text in the page's charset, or in the datadase's charset
 *
 * @param string the text
 * @param string the page's encoding
 * @param bool is it for the database ?
 * @return string the encoded text
 */
function _convert($text, $fromencoding, $database = FALSE)
{
    $cp1252_map = array(
        "\xc2\x80" => "\xe2\x82\xac", /* EURO SIGN */
        "\xc2\x82" => "\xe2\x80\x9a", /* SINGLE LOW-9 QUOTATION MARK */
        "\xc2\x83" => "\xc6\x92",    /* LATIN SMALL LETTER F WITH HOOK */
        "\xc2\x84" => "\xe2\x80\x9e", /* DOUBLE LOW-9 QUOTATION MARK */
        "\xc2\x85" => "\xe2\x80\xa6", /* HORIZONTAL ELLIPSIS */
        "\xc2\x86" => "\xe2\x80\xa0", /* DAGGER */
        "\xc2\x87" => "\xe2\x80\xa1", /* DOUBLE DAGGER */
        "\xc2\x88" => "\xcb\x86",    /* MODIFIER LETTER CIRCUMFLEX ACCENT */
        "\xc2\x89" => "\xe2\x80\xb0", /* PER MILLE SIGN */
        "\xc2\x8a" => "\xc5\xa0",    /* LATIN CAPITAL LETTER S WITH CARON */
        "\xc2\x8b" => "\xe2\x80\xb9", /* SINGLE LEFT-POINTING ANGLE QUOTATION */
        "\xc2\x8c" => "\xc5\x92",    /* LATIN CAPITAL LIGATURE OE */
        "\xc2\x8e" => "\xc5\xbd",    /* LATIN CAPITAL LETTER Z WITH CARON */
        "\xc2\x91" => "\xe2\x80\x98", /* LEFT SINGLE QUOTATION MARK */
        "\xc2\x92" => "\xe2\x80\x99", /* RIGHT SINGLE QUOTATION MARK */
        "\xc2\x93" => "\xe2\x80\x9c", /* LEFT DOUBLE QUOTATION MARK */
        "\xc2\x94" => "\xe2\x80\x9d", /* RIGHT DOUBLE QUOTATION MARK */
        "\xc2\x95" => "\xe2\x80\xa2", /* BULLET */
        "\xc2\x96" => "\xe2\x80\x93", /* EN DASH */
        "\xc2\x97" => "\xe2\x80\x94", /* EM DASH */

        "\xc2\x98" => "\xcb\x9c",    /* SMALL TILDE */
        "\xc2\x99" => "\xe2\x84\xa2", /* TRADE MARK SIGN */
        "\xc2\x9a" => "\xc5\xa1",    /* LATIN SMALL LETTER S WITH CARON */
        "\xc2\x9b" => "\xe2\x80\xba", /* SINGLE RIGHT-POINTING ANGLE QUOTATION*/
        "\xc2\x9c" => "\xc5\x93",    /* LATIN SMALL LIGATURE OE */
        "\xc2\x9e" => "\xc5\xbe",    /* LATIN SMALL LETTER Z WITH CARON */
        "\xc2\x9f" => "\xc5\xb8"      /* LATIN CAPITAL LETTER Y WITH DIAERESIS*/
    );

    if ($database) { 
        if ($fromencoding != "ISO-8859-1" && $fromencoding != "ISO-8859-15") {
            //return iconv($fromencoding, "ISO-8859-15", $text);
            return  utf8_decode( strtr($text, array_flip($cp1252_map)) );
        } 
        else {
            return $text;
        }
    }
    else {
        if ($fromencoding != TEMPLATES_DEFAULT_CHARSET) {
            //return iconv($fromencoding, TEMPLATES_DEFAULT_CHARSET, $text);
            return strtr(utf8_encode($text), $cp1252_map);
        } 
        else {
            return $text;
        }
{'fraction_non_alphanumeric': 0.13480968858131487, 'fraction_numerical': 0.03474048442906574, 'mean_word_length': 2.542156862745098, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2719135', 'n_tokens_mistral': 2506, 'n_tokens_neox': 2301, 'n_words': 715}