<commit_before>package chat.rocket.android.widget.emoji

import android.content.Context
import android.content.SharedPreferences
import android.graphics.Typeface
import android.os.Build
import org.json.JSONArray
import org.json.JSONObject
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
import java.util.*
import java.util.regex.Pattern

object EmojiRepository {
    private val shortNameToUnicode = HashMap<String, String>()
    private val SHORTNAME_PATTERN = Pattern.compile(":([-+\\w]+):")
    private val ALL_EMOJIS = mutableListOf<Emoji>()
    private lateinit var preferences: SharedPreferences
    internal lateinit var cachedTypeface: Typeface

    fun load(context: Context, path: String = "emoji.json") {
        preferences = context.getSharedPreferences("emoji", Context.MODE_PRIVATE)
        ALL_EMOJIS.clear()
        cachedTypeface = Typeface.createFromAsset(context.assets, "fonts/emojione-android.ttf")
        val stream = context.assets.open(path)
        val emojis = loadEmojis(stream)
        emojis.forEach {
            val unicodeIntList = mutableListOf<Int>()
            it.unicode.split("-").forEach {
                val value = it.toInt(16)
                if (value >= 0x10000) {
                    val surrogatePair = calculateSurrogatePairs(value)
                    unicodeIntList.add(surrogatePair.first)
                    unicodeIntList.add(surrogatePair.second)
                } else {
                    unicodeIntList.add(value)
                }
            }
            val unicodeIntArray = unicodeIntList.toIntArray()
            val unicode = String(unicodeIntArray, 0, unicodeIntArray.size)
            ALL_EMOJIS.add(it.copy(unicode = unicode))
            shortNameToUnicode.apply { put(it.shortname, unicode) }
        }
    }

    /**
     * Get all loaded emojis as list of Emoji objects.
     *
     * @return All emojis for all categories.
     */
    fun getAll() = ALL_EMOJIS

    /**
     * Get all emojis for a given category.
     *
     * @param category Emoji category such as: PEOPLE, NATURE, ETC
     *
     * @return All emoji from specified category
     */
    fun getEmojisByCategory(category: EmojiCategory): List<Emoji> {
        return ALL_EMOJIS.filter { it.category.toLowerCase() == category.name.toLowerCase() }
    }

    /**<commit_msg>Look alternate shortnames when looking for matching unicode<commit_after>package chat.rocket.android.widget.emoji

import android.content.Context
import android.content.SharedPreferences
import android.graphics.Typeface
import android.os.Build
import org.json.JSONArray
import org.json.JSONObject
import java.io.BufferedReader
import java.io.InputStream
import java.io.InputStreamReader
import java.util.*
import java.util.regex.Pattern

object EmojiRepository {
    private val shortNameToUnicode = HashMap<String, String>()
    private val SHORTNAME_PATTERN = Pattern.compile(":([-+\\w]+):")
    private val ALL_EMOJIS = mutableListOf<Emoji>()
    private lateinit var preferences: SharedPreferences
    internal lateinit var cachedTypeface: Typeface

    fun load(context: Context, path: String = "emoji.json") {
        preferences = context.getSharedPreferences("emoji", Context.MODE_PRIVATE)
        ALL_EMOJIS.clear()
        cachedTypeface = Typeface.createFromAsset(context.assets, "fonts/emojione-android.ttf")
        val stream = context.assets.open(path)
        val emojis = loadEmojis(stream)
        emojis.forEach {
            val unicodeIntList = mutableListOf<Int>()
            it.unicode.split("-").forEach {
                val value = it.toInt(16)
                if (value >= 0x10000) {
                    val surrogatePair = calculateSurrogatePairs(value)
                    unicodeIntList.add(surrogatePair.first)
                    unicodeIntList.add(surrogatePair.second)
                } else {
                    unicodeIntList.add(value)
                }
            }
            val unicodeIntArray = unicodeIntList.toIntArray()
            val unicode = String(unicodeIntArray, 0, unicodeIntArray.size)
            ALL_EMOJIS.add(it.copy(unicode = unicode))
            shortNameToUnicode.apply {
                put(it.shortname, unicode)
                it.shortnameAlternates.forEach { alternate -> put(alternate, unicode) }
            }
        }
    }

    /**
     * Get all loaded emojis as list of Emoji objects.
     *
     * @return All emojis for all categories.
     */
    fun getAll() = ALL_EMOJIS

    /**
     * Get all emojis for a given category.
     *
     * @param category Emoji category such as: PEOPLE, NATURE, ETC
     *
     * @return All emoji from specified category
     */
    fun getEmojisByCategory(category: EmojiCategory): List<Emoji> {
        return ALL_EMOJIS.filter { it.category.toLowerCase() == category.name.toLowerCase() }
    }

    /**
{'fraction_non_alphanumeric': 0.09405737704918032, 'fraction_numerical': 0.0036885245901639345, 'mean_word_length': 2.5064655172413794, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6648324', 'n_tokens_mistral': 1437, 'n_tokens_neox': 1381, 'n_words': 305}