<commit_before><?php

namespace Chefkoch\Morphoji;

/**
 * Class Converter
 *
 * Converts texts with Unicode Emoji (https://en.wikipedia.org/wiki/Emoji,
 * http://unicode.org/emoji/index.html) to texts where all Emoji found are
 * replaced with latin1 based placeholders.
 *
 * Default placeholders are of the form `:emoji-HEX_VALUE:` where `HEX_VALUE` is
 * the hex value of the emoji's unicode character.
 *
 * @package Chefkoch\Morphoji
 */
class Converter
{

    /** @var string */
    private static $emojiPattern;
    /** @var string */
    private static $entityPattern;

    /**
     * Converts all unicode emoji in the given text into placeholders.
     *
     * Returns the text with the morphed emoji or null if there was a problem.
     *
     * @param string $text
     * @return string|null
     */
    public function emojiToEntities($text)
    {
        $pattern = $this->getEmojiPattern();

        $latin1Text = preg_replace_callback($pattern, function ($match) {
            $char = array_pop($match);
            $utf32 = mb_convert_encoding($char, 'UTF-32', 'UTF-8');
            $hex = ltrim(bin2hex($utf32), '0');

            return $this->getHtmlEntity($hex);
        }, $text);

        return $latin1Text;
    }

    /**
     * Converts all placeholders in the given text into unicode emoji.
     *
     * Returns the text with the morphed placeholders or null if there was a
     * problem.
     *
     * @param string $text
     * @return string|null
     */
    public function entitiesToEmoji($text)
    {
        $hexPattern = '/^' . $this->getHtmlEntity('([\w\d]{2,8})') . '$/i';

        $unicodeText = preg_replace_callback($this->getEntityPattern(),
            function ($match) use ($hexPattern) {
                $entity = array_pop($match);
                preg_match($hexPattern, $entity, $matches);
                $hex4 = str_pad(array_pop($matches), 8, '0', STR_PAD_LEFT);

                return mb_convert_encoding(hex2bin($hex4), 'UTF-8', 'UTF-32');
            }
        , $text);

        return $unicodeText;
    }

    /**
     * Returns the given hex string as html entity.
     *
     * @param string $hex
     * @return string
     */
    private function getHtmlEntity($hex)
    {
        return "&#x$hex;";
    }

    /**
     * Returns a regular expression pattern to detect emoji characters.
     *
     * @return string
     */
    private function getEmojiPattern()
    {
        if (null === self::$emojiPattern) {
            $codeString = '';

            foreach ($this->getEmojiCodeList() as $code) {
                if (is_array($code)) {

                    $first = dechex(array_shift($code));
                    $last  = dechex(array_pop($code));
                    $codeString .= '\x{' . $first . '}-\x{' . $last . '}';

                } else {
                    $codeString .= '\x{' . dechex($code) . '}';
                }
            }

            self::$emojiPattern = "/[$codeString]/u";
        }

        return self::$emojiPattern;
    }

    /**
     * Returns a regex pattern to detect emoji html entities.
     *
     * @return string
     */
    private function getEntityPattern()
    {
        if (null === self::$entityPattern) {
            $entities = [];

            foreach ($this->getEmojiCodeList() as $code) {
                if (!is_array($code)) {
                    $entities[] = $this->getHtmlEntity(dechex($code));
                } else {
                    foreach($code as $c) {
                        $entities[] = $this->getHtmlEntity(dechex($c));
                    }
                }
            }

            self::$entityPattern = '/' . implode('|', $entities) . '/i';
        }

        return self::$entityPattern;
    }

    /**
     * Returns an array with all unicode values for emoji characters.
     *
     * This is a function so the array can be defined with a mix of hex values
     * and range() calls to conveniently meintain the array with information
     * from the official Unicode tables (where values are given in hex as well).
     *
     * With PHP > 5.6 this could be done in class variable, maybe even a
     * constant.
     *
     * @return array
     */
    private function getEmojiCodeList()
    {
        return [
            // Various 'older' charactes, dingbats etc. which over time have
            // received an additional emoji representation.
            0x203c,
            0x2049,
            0x2122,<commit_msg>Fix typo<commit_after><?php

namespace Chefkoch\Morphoji;

/**
 * Class Converter
 *
 * Converts texts with Unicode Emoji (https://en.wikipedia.org/wiki/Emoji,
 * http://unicode.org/emoji/index.html) to texts where all Emoji found are
 * replaced with latin1 based placeholders.
 *
 * Default placeholders are of the form `:emoji-HEX_VALUE:` where `HEX_VALUE` is
 * the hex value of the emoji's unicode character.
 *
 * @package Chefkoch\Morphoji
 */
class Converter
{

    /** @var string */
    private static $emojiPattern;
    /** @var string */
    private static $entityPattern;

    /**
     * Converts all unicode emoji in the given text into placeholders.
     *
     * Returns the text with the morphed emoji or null if there was a problem.
     *
     * @param string $text
     * @return string|null
     */
    public function emojiToEntities($text)
    {
        $pattern = $this->getEmojiPattern();

        $latin1Text = preg_replace_callback($pattern, function ($match) {
            $char = array_pop($match);
            $utf32 = mb_convert_encoding($char, 'UTF-32', 'UTF-8');
            $hex = ltrim(bin2hex($utf32), '0');

            return $this->getHtmlEntity($hex);
        }, $text);

        return $latin1Text;
    }

    /**
     * Converts all placeholders in the given text into unicode emoji.
     *
     * Returns the text with the morphed placeholders or null if there was a
     * problem.
     *
     * @param string $text
     * @return string|null
     */
    public function entitiesToEmoji($text)
    {
        $hexPattern = '/^' . $this->getHtmlEntity('([\w\d]{2,8})') . '$/i';

        $unicodeText = preg_replace_callback($this->getEntityPattern(),
            function ($match) use ($hexPattern) {
                $entity = array_pop($match);
                preg_match($hexPattern, $entity, $matches);
                $hex4 = str_pad(array_pop($matches), 8, '0', STR_PAD_LEFT);

                return mb_convert_encoding(hex2bin($hex4), 'UTF-8', 'UTF-32');
            }
        , $text);

        return $unicodeText;
    }

    /**
     * Returns the given hex string as html entity.
     *
     * @param string $hex
     * @return string
     */
    private function getHtmlEntity($hex)
    {
        return "&#x$hex;";
    }

    /**
     * Returns a regular expression pattern to detect emoji characters.
     *
     * @return string
     */
    private function getEmojiPattern()
    {
        if (null === self::$emojiPattern) {
            $codeString = '';

            foreach ($this->getEmojiCodeList() as $code) {
                if (is_array($code)) {

                    $first = dechex(array_shift($code));
                    $last  = dechex(array_pop($code));
                    $codeString .= '\x{' . $first . '}-\x{' . $last . '}';

                } else {
                    $codeString .= '\x{' . dechex($code) . '}';
                }
            }

            self::$emojiPattern = "/[$codeString]/u";
        }

        return self::$emojiPattern;
    }

    /**
     * Returns a regex pattern to detect emoji html entities.
     *
     * @return string
     */
    private function getEntityPattern()
    {
        if (null === self::$entityPattern) {
            $entities = [];

            foreach ($this->getEmojiCodeList() as $code) {
                if (!is_array($code)) {
                    $entities[] = $this->getHtmlEntity(dechex($code));
                } else {
                    foreach($code as $c) {
                        $entities[] = $this->getHtmlEntity(dechex($c));
                    }
                }
            }

            self::$entityPattern = '/' . implode('|', $entities) . '/i';
        }

        return self::$entityPattern;
    }

    /**
     * Returns an array with all unicode values for emoji characters.
     *
     * This is a function so the array can be defined with a mix of hex values
     * and range() calls to conveniently maintain the array with information
     * from the official Unicode tables (where values are given in hex as well).
     *
     * With PHP > 5.6 this could be done in class variable, maybe even a
     * constant.
     *
     * @return array
     */
    private function getEmojiCodeList()
    {
        return [
            // Various 'older' charactes, dingbats etc. which over time have
            // received an additional emoji representation.
            0x203c,
            0x2049,
            0x2122,
{'fraction_non_alphanumeric': 0.1251259093452714, 'fraction_numerical': 0.008505875769446, 'mean_word_length': 1.806532663316583, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2594477', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2568, 'n_words': 698}