<commit_before>package net.milosvasic.pussycat.android.data.parser

import com.android.ddmlib.Log
import com.android.ddmlib.logcat.LogCatMessage
import net.milosvasic.pussycat.android.data.AndroidLogCatMessage
import java.util.*
import java.util.regex.Matcher
import java.util.LinkedHashMap

class LogCatMessageParser {

    var lastMessage: AndroidLogCatMessage? = null
    private val listeners = HashSet<LogCatMessageParserMatcherListener>()

    companion object {
        val UNKNOWN_VALUE = "unknown"
        val TERMINAL_DUMP_PATTERN = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\w)\\s+(.+?):(.+?)"
        val TERMINAL_DUMP_PATTERN_CROPPED = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\w)\\s+(.+?):"
        val ANDROID_STUDIO_DUMP_PATTERN = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)-(\\d+).+?\\s+(\\w)/(.+?):(.+?)"
        val ANDROID_STUDIO_PATTERN_CROPPED = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)-(\\d+).+?\\s+(\\w)/(.+?):"
        val ANDROID_STUDIO_PATTERN_STACKTRACE = "\\s+(.+?)"
        val ACER_Z520_PATTERN = "(\\w)/(\\w+).+?(\\w+).:\\s+(.+?)"
        val DEFAULT_HEAD_PATTERN = "^\\[\\s(\\d\\d-\\d\\d\\s\\d\\d:\\d\\d:\\d\\d\\.\\d+)\\s+(\\d*):\\s*(\\S+)\\s([VDIWEAF])/(.*[^\\s])\\s+\\]$"

        fun getIdentifier(message: AndroidLogCatMessage): String {
            return "${message.time}_${message.pid}_${message.tid}"
        }
    }

    private val terminalDumpPattern = LogCatMessagePattern(TERMINAL_DUMP_PATTERN, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            val logMessage = if (matcher.groupCount() == 7) {
                matcher.group(7)
            } else {
                ""
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    logMessage.trim()
            )
        }
    })
    private val terminalDumpPatternCropped = LogCatMessagePattern(TERMINAL_DUMP_PATTERN_CROPPED, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    ""
            )
        }
    })

    private val androidStudioDumpPattern = LogCatMessagePattern(ANDROID_STUDIO_DUMP_PATTERN, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            val logMessage = if (matcher.groupCount() == 7) {
                matcher.group(7)
            } else {
                ""
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    logMessage.trim()
            )
        }
    })
    private val androidStudioDumpPatternCropped = LogCatMessagePattern(ANDROID_STUDIO_PATTERN_CROPPED, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    ""
            )
        }
    })
    private val androidStudioDumpPatternStacktrace = LogCatMessagePattern(ANDROID_STUDIO_PATTERN_STACKTRACE, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = lastMessage?.logLevel
            if (logLevel == null) {
                logLevel = Log.LogLevel.VERBOSE
            }
            var pid = lastMessage?.pid
            if (pid == null) {
                pid = -1
            }
            var tid = lastMessage?.tid
            if (tid == null) {
                tid = -1
            }
            var appName = lastMessage?.appName
            if (appName == null) {
                appName = "app $UNKNOWN_VALUE"
            }
            var tag = lastMessage?.tag
            if (tag == null) {
                tag = "tag $UNKNOWN_VALUE"
            }
            var time = lastMessage?.time
            if (time == null) {
                time = "time $UNKNOWN_VALUE"
            }
            return AndroidLogCatMessage(
                    logLevel,
                    pid,
                    tid,
                    appName,
                    tag,
                    time,
                    matcher.group(1).trim()
            )
        }
    })

    private val acerZ520Pattern = LogCatMessagePattern(ACER_Z520_PATTERN, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(1))
            if (logLevel == null && matcher.group(1) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            val logMessage = if (matcher.groupCount() == 4) {
                matcher.group(4)
            } else {
                ""
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    -1,
                    matcher.group(2).trim(),
                    matcher.group(2).trim(),
                    UNKNOWN_VALUE,
                    logMessage.trim()
            )
        }
    })

    private val defaultLogCatMessageHeaderMessageObtain = DefaultLogCatMessageHeaderMessageObtain()
    private val defaultHeadPattern = DefaultLogCatMessageHeaderPattern(DEFAULT_HEAD_PATTERN, defaultLogCatMessageHeaderMessageObtain)

    private val patterns = listOf(
            terminalDumpPattern,
            terminalDumpPatternCropped,
            androidStudioDumpPattern,
            androidStudioDumpPatternCropped,
            androidStudioDumpPatternStacktrace,
            defaultHeadPattern,
            acerZ520Pattern
    )

    fun processLogLines(lines: Array<String>): Collection<AndroidLogCatMessage> {
        val messages = Collections.synchronizedMap(LinkedHashMap<String, AndroidLogCatMessage>())
        for (line in lines) {
            if (line.isEmpty()) {
                continue
            }
            var matched = false
            for (logCatPattern in patterns) {
                val pattern = logCatPattern.compile()<commit_msg>Pussycat:  Writing parser test, extending parser to support more log formats.<commit_after>package net.milosvasic.pussycat.android.data.parser

import com.android.ddmlib.Log
import com.android.ddmlib.logcat.LogCatMessage
import net.milosvasic.pussycat.android.data.AndroidLogCatMessage
import java.util.*
import java.util.regex.Matcher
import java.util.LinkedHashMap

class LogCatMessageParser {

    var lastMessage: AndroidLogCatMessage? = null
    private val listeners = HashSet<LogCatMessageParserMatcherListener>()

    companion object {
        val UNKNOWN_VALUE = "unknown"
        val TERMINAL_DUMP_PATTERN = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\w)\\s+(.+?):(.+?)"
        val TERMINAL_DUMP_PATTERN_CROPPED = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\w)\\s+(.+?):"
        val ANDROID_STUDIO_DUMP_PATTERN = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)-(\\d+).+?\\s+(\\w)/(.+?):(.+?)"
        val ANDROID_STUDIO_PATTERN_CROPPED = "(\\d+-\\d+)\\s+(\\d+:\\d+:\\d+.\\d+)\\s+(\\d+)-(\\d+).+?\\s+(\\w)/(.+?):"
        val ANDROID_STUDIO_PATTERN_STACKTRACE = "\\s+(.+?)"
        val ACER_Z520_PATTERN = "(\\w)/(\\w+).+?(\\w+).:\\s+(.+?)"
        val ACER_Z520_PATTERN2 = "(\\w)/(\\s+).+?(\\w+).:\\s+(.+?)"
        val DEFAULT_HEAD_PATTERN = "^\\[\\s(\\d\\d-\\d\\d\\s\\d\\d:\\d\\d:\\d\\d\\.\\d+)\\s+(\\d*):\\s*(\\S+)\\s([VDIWEAF])/(.*[^\\s])\\s+\\]$"

        fun getIdentifier(message: AndroidLogCatMessage): String {
            return "${message.time}_${message.pid}_${message.tid}"
        }
    }

    private val terminalDumpPattern = LogCatMessagePattern(TERMINAL_DUMP_PATTERN, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            val logMessage = if (matcher.groupCount() == 7) {
                matcher.group(7)
            } else {
                ""
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    logMessage.trim()
            )
        }
    })
    private val terminalDumpPatternCropped = LogCatMessagePattern(TERMINAL_DUMP_PATTERN_CROPPED, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    ""
            )
        }
    })

    private val androidStudioDumpPattern = LogCatMessagePattern(ANDROID_STUDIO_DUMP_PATTERN, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            val logMessage = if (matcher.groupCount() == 7) {
                matcher.group(7)
            } else {
                ""
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    logMessage.trim()
            )
        }
    })
    private val androidStudioDumpPatternCropped = LogCatMessagePattern(ANDROID_STUDIO_PATTERN_CROPPED, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(5))
            if (logLevel == null && matcher.group(5) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    Integer.valueOf(matcher.group(4).trim()),
                    matcher.group(6).trim(),
                    matcher.group(6).trim(),
                    matcher.group(2).trim(),
                    ""
            )
        }
    })
    private val androidStudioDumpPatternStacktrace = LogCatMessagePattern(ANDROID_STUDIO_PATTERN_STACKTRACE, object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = lastMessage?.logLevel
            if (logLevel == null) {
                logLevel = Log.LogLevel.VERBOSE
            }
            var pid = lastMessage?.pid
            if (pid == null) {
                pid = -1
            }
            var tid = lastMessage?.tid
            if (tid == null) {
                tid = -1
            }
            var appName = lastMessage?.appName
            if (appName == null) {
                appName = "app $UNKNOWN_VALUE"
            }
            var tag = lastMessage?.tag
            if (tag == null) {
                tag = "tag $UNKNOWN_VALUE"
            }
            var time = lastMessage?.time
            if (time == null) {
                time = "time $UNKNOWN_VALUE"
            }
            return AndroidLogCatMessage(
                    logLevel,
                    pid,
                    tid,
                    appName,
                    tag,
                    time,
                    matcher.group(1).trim()
            )
        }
    })

    val acerMessageObtain = object : LogCatMessageObtain {
        override fun getMessage(matcher: Matcher): AndroidLogCatMessage {
            var logLevel = Log.LogLevel.getByLetterString(matcher.group(1))
            if (logLevel == null && matcher.group(1) == "F") {
                logLevel = Log.LogLevel.ASSERT
            }
            val logMessage = if (matcher.groupCount() == 4) {
                matcher.group(4)
            } else {
                ""
            }
            return AndroidLogCatMessage(
                    logLevel,
                    Integer.valueOf(matcher.group(3).trim()),
                    -1,
                    matcher.group(2).trim(),
                    matcher.group(2).trim(),
                    UNKNOWN_VALUE,
                    logMessage.trim()
            )
        }
    }

    private val acerZ520Pattern = LogCatMessagePattern(ACER_Z520_PATTERN, acerMessageObtain)
    private val acerZ520Pattern2 = LogCatMessagePattern(ACER_Z520_PATTERN2, acerMessageObtain)

    private val defaultLogCatMessageHeaderMessageObtain = DefaultLogCatMessageHeaderMessageObtain()
    private val defaultHeadPattern = DefaultLogCatMessageHeaderPattern(DEFAULT_HEAD_PATTERN, defaultLogCatMessageHeaderMessageObtain)

    private val patterns = listOf(
            terminalDumpPattern,
            terminalDumpPatternCropped,
            androidStudioDumpPattern,
            androidStudioDumpPatternCropped,
            androidStudioDumpPatternStacktrace,
            defaultHeadPattern,
            acerZ520Pattern,
            acerZ520Pattern2
    )

    fun processLogLines(lines: Array<String>): Collection<AndroidLogCatMessage> {
        val messages = Collections.synchronizedMap(LinkedHashMap<String, AndroidLogCatMessage>())
        for (line in lines) {
            if (line.isEmpty()) {
                continue
            }
            var matched = false
            for (logCatPattern in patterns) {
                val pattern = logCatPattern.compile()
{'fraction_non_alphanumeric': 0.13370233702337023, 'fraction_numerical': 0.007749077490774908, 'mean_word_length': 1.639344262295082, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '684294', 'n_tokens_mistral': 4879, 'n_tokens_neox': 4636, 'n_words': 703}