<commit_before>package me.mrkirby153.KirBot.redis.messaging

import me.mrkirby153.KirBot.database.api.ServerMessage
import me.mrkirby153.KirBot.redis.RedisConnector
import org.json.JSONObject
import org.json.JSONTokener

class MessageDataStore {

    private val LIST_KEY = "messages"

    fun getMessageContent(id: String, callback: (msg: Message?) -> Unit) {
        RedisConnector.get().use {
            val listSize = it.llen(LIST_KEY)

            val list = it.lrange(LIST_KEY, 0, listSize).map { JSONObject(JSONTokener(it)) }.map { decode(it) }

            val messages = list.filter { it.id == id }

            if (messages.isEmpty()) {
                ServerMessage.get(id).queue {
                    callback.invoke(Message(it.id ?: "-1", it.serverId, it.authorId, it.channelId, it.content))
                }
                return
            }

            callback.invoke(messages[messages.lastIndex])
        }
    }

    fun pushMessage(message: net.dv8tion.jda.core.entities.Message) {
        RedisConnector.get().use {
            val json = encode(Message(message.id, message.guild.id, message.author.id, message.channel.id, message.content, message.creationTime.toEpochSecond()))
            it.rpush(LIST_KEY, json.toString())
        }
    }

    fun deleteMessage(messageId: String, callback: ((msg: String) -> Unit)?) {
        RedisConnector.get().use {
            val msgs = it.llen(LIST_KEY)
            val msgList = it.lrange(LIST_KEY, 0, msgs)
            msgList.forEach { msg ->
                val decoded = decode(JSONObject(JSONTokener(msg)))
                if (decoded.id == messageId) {
                    callback?.invoke(decoded.message)
                    it.lrem(LIST_KEY, 0, msg)
                    return
                }
            }
        }
        ServerMessage.delete(messageId).queue {
            callback?.invoke(it.content)
        }
    }

    fun bulkDelete(messageIds: Array<String>, callback: (() -> Unit)?) {
        val deleted = mutableListOf<String>()
        RedisConnector.get().use {
            val msgList = it.lrange(LIST_KEY, 0, it.llen(LIST_KEY))
            msgList.forEach { msg ->
                val decoded = decode(JSONObject(JSONTokener(msg)))
                deleted.add(decoded.id)
                if (decoded.id in messageIds)
                    it.lrem(LIST_KEY, 0, msg)
            }
        }
        val toDelete = messageIds.clone().toMutableList()
        toDelete.removeAll(deleted)
        println(toDelete)
        if (toDelete.isNotEmpty())
            ServerMessage.bulkDelete(toDelete).queue {
                callback?.invoke()
            }
        else
            callback?.invoke()
    }


    fun encode(message: Message): JSONObject = JSONObject().apply {
        put("id", message.id)
        put("author", message.author)
        put("channel", message.channel)
        put("message", message.message)
        put("server", message.serverId)
        put("time", message.time)
    }

    fun decode(obj: JSONObject): Message = Message(obj.getString("id"), obj.getString("server"), obj.getString("author"), obj.getString("channel"), obj.getString("message"), obj.optLong("time"))


    data class Message(val id: String, val serverId: String, val author: String, val channel: String, val message: String, val time: Long? = System.currentTimeMillis())
}<commit_msg>Updated how messages are stored in the redis queue<commit_after>package me.mrkirby153.KirBot.redis.messaging

import me.mrkirby153.KirBot.database.api.ServerMessage
import me.mrkirby153.KirBot.redis.RedisConnector
import org.json.JSONObject
import org.json.JSONTokener

class MessageDataStore {

    private val LIST_KEY = "messages"

    private val DATA_KEY_PREFIX = "message"

    fun getMessageContent(id: String, callback: (msg: Message?) -> Unit) {
        RedisConnector.get().use {
            val msg = getMessageFromRedis(id)
            if (msg != null)
                callback.invoke(msg)
            else {
                ServerMessage.get(id).queue {
                    callback.invoke(Message(it.id ?: "-1", it.serverId, it.authorId, it.channelId,
                            it.content))
                }
            }
        }
    }

    fun storeMessage(message: net.dv8tion.jda.core.entities.Message): String {
        val key = "$DATA_KEY_PREFIX:${message.id}"
        RedisConnector.get().use {
            it.set(key, encode(Message(message)).toString())
            return key
        }
    }

    fun getMessageFromRedis(key: String): Message? {
        RedisConnector.get().use {
            val json = it.get("$DATA_KEY_PREFIX:$key") ?: return null
            val obj = JSONObject(JSONTokener(json))
            return decode(obj)
        }
    }

    fun pushMessage(message: net.dv8tion.jda.core.entities.Message) {
        RedisConnector.get().use {
            it.rpush(LIST_KEY, storeMessage(message))
        }
    }

    fun deleteMessage(messageId: String, callback: ((msg: String) -> Unit)?) {
        val msg = getMessageFromRedis(messageId)
        if (msg != null) { // Message is queued in redis
            RedisConnector.get().use {
                it.lrem(LIST_KEY, 0, "$DATA_KEY_PREFIX:$messageId")
                it.del("$DATA_KEY_PREFIX:$messageId")
                callback?.invoke(msg.message)
                return
            }
        } else {
            ServerMessage.delete(messageId).queue {
                callback?.invoke(it.content)
            }
        }
    }

    fun bulkDelete(messageIds: Array<String>, callback: (() -> Unit)?) {
        val deleted = mutableListOf<String>()
        RedisConnector.get().use {
            messageIds.forEach { id ->
                if (getMessageFromRedis(id) != null) {
                    it.lrem(LIST_KEY, 0, "$DATA_KEY_PREFIX:$id")
                    it.del("$DATA_KEY_PREFIX:$id")
                    deleted.add(id)
                }
            }
        }
        val toDelete = messageIds.clone().filter { it !in deleted }
        if (toDelete.isNotEmpty()) {
            ServerMessage.bulkDelete(toDelete).queue {
                callback?.invoke()
            }
        } else {
            callback?.invoke()
        }
    }


    private fun encode(message: Message): JSONObject = JSONObject().apply {
        put("id", message.id)
        put("author", message.author)
        put("channel", message.channel)
        put("message", message.message)
        put("server", message.serverId)
        put("time", message.time)
    }

    private fun decode(obj: JSONObject): Message = Message(obj.getString("id"), obj.getString("server"),
            obj.getString("author"), obj.getString("channel"), obj.getString("message"),
            obj.optLong("time"))


    data class Message(val id: String, val serverId: String, val author: String,
                       val channel: String, val message: String,
                       val time: Long? = System.currentTimeMillis()) {
        constructor(msg: net.dv8tion.jda.core.entities.Message) : this(msg.id, msg.guild.id,
                msg.author.id, msg.channel.id, msg.content, msg.creationTime.toEpochSecond())
    }
}
{'fraction_non_alphanumeric': 0.12473838426119715, 'fraction_numerical': 0.004325380214873727, 'mean_word_length': 2.0231969633066216, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4742267', 'n_tokens_mistral': 2108, 'n_tokens_neox': 2038, 'n_words': 383}