<commit_before>
package fgc.formatter

import fgc.scraper.VideoItem
import fgc.scraper.YouTubeChannel

import java.io._
import scala.util.matching.Regex

import scalaj.http._
import org.json4s._
import org.json4s.native.Serialization

case class VideoData(
    id: String,
    timestamp: String,
    game: String,
    players: List[String],
    characters: List[List[String]]
){
    val tuple: List[Any] = List(
        timestamp, id, game, players, characters
    )
}

trait Parser {
    val channel: YouTubeChannel
    def parseVideo(videoItem: VideoItem): Option[VideoData]
    def loadVideos(): List[VideoData] = {
        channel.loadFile.values.flatMap(parseVideo).toList
    }

    val rPlayer = "([\\w\\.\\-\\| ]+) "
    val rCharacter = "(?:\\(|\\[) *([\\w\\. ]+) *(?:\\)|\\]) "
    val rVersus = "(?:Vs|vs)\\.? "
    val rGameMap: Map[String, String]
    def rGame(): String = {
        val rGames = rGameMap.values.mkString("|")
        s" *($rGames) *"
    }

    def fixGame(rawGame: String): String = {
        // todo improve this
        var matchingKey = ""
        rGameMap.foreach { case (key, value) =>
            val matchMaybe = value.r.findFirstIn(rawGame)
            matchMaybe match {
                case Some(name) =>
                    matchingKey = key
                case None =>
                    // do nothing
            }
        }
        matchingKey
    }
    def fixCharacters(char1: String, char2: String): List[List[String]] = {
        List(List(char1), List(char2))
    }
    def fixPlayers(player1: String, player2: String): List[String] = {
        List(player1, player2)
    }
}

object YogaFlameParser extends Parser {
    val channel = YouTubeChannel.YogaFlame

    val rGameMap = Map(
        "SF5" -> "SF5|SFV|Beta SFV",
        "USF4" -> "USF4",
        "SSF4AE2012" -> "(?:Arcade Edition|AE)(?: Version)? +2012",
        "SSF4AE" -> "Arcade Edition|AE",
        "SF3" -> "SF3",
        "SFxT" -> "SFxT",
        "TTT2" -> "Tekken Tag Tournament 2"
    )
    val regex: Regex = {
        val rRounds = "(?:X[0-9] )?"
        val rResolution = " *(?:1080p|720p)"
        (
            rRounds +
            rPlayer +
            rCharacter +
            rVersus +
            rPlayer +
            rCharacter +
            rGame +
            rResolution
        ).r
    }

    def parseVideo(videoItem: VideoItem): Option[VideoData] = {
        val matchMaybe = regex.findFirstMatchIn(videoItem.title)
        matchMaybe match {
            case Some(matchRes) => {
                val matches = matchRes.subgroups
                Option(new VideoData(
                    videoItem.id,
                    videoItem.timestamp,
                    fixGame(matches(4)),
                    fixPlayers(matches(0), matches(2)),
                    fixCharacters(matches(1), matches(3))
                ))
            }
            case None => {
                // failed regex, maybe log?
                None
            }
        }
    }
}

object OlympicGamingParser extends Parser {
    val channel = YouTubeChannel.OlympicGaming

    val rGameMap = Map(
        "SF5" -> "Street Fighter (?:5 *\\/? *V|5|V)|SFV",
        "SFxT" -> "Street Fighter X Tekken",
        "P4AU" -> "Persona 4 Arena Ultimax",
        "MKX" -> "Mortal Kombat X",
        "SC5" -> "SoulCalibur 5\\/V",
        "DOA5" -> "Dead Or Alive 5 Last Round",
        "Smash4" -> "Super Smash Bros Wii U",
        "GGXrd" -> "Guilty Gear Xrd",
        "KI" -> "Killer Instinct"
    )
    // val rEndtag = " *(?:[\\w ]*-? *Gameplay).*"
    val rEndtag = "(?:Wii|Xbox|PS4|-? ?Gameplay).*"
    val regex1 = (
        rPlayer +
        rCharacter +
        rVersus +
        rPlayer +
        rCharacter +
        rGame +
        rEndtag
    ).r
    val regex2 = (
        rGame + ": " +
        rPlayer +
        rCharacter +
        rVersus +
        rPlayer +
        rCharacter +
        rEndtag
    ).r

    def parseVideo(videoItem: VideoItem): Option[VideoData] = {
        var res: Option[VideoData] = None
        val matchMaybe1 = regex1.findFirstMatchIn(videoItem.title)
        matchMaybe1 match {
            case Some(matchRes) => {
                val matches = matchRes.subgroups
                res = Option(new VideoData(
                    videoItem.id,
                    videoItem.timestamp,
                    fixGame(matches(4)),
                    fixPlayers(matches(0), matches(2)),
                    fixCharacters(matches(1), matches(3))
                ))
            }
            case None => {}
        }
        val matchMaybe2 = regex2.findFirstMatchIn(videoItem.title)
        matchMaybe2 match {
            case Some(matchRes) => {
                val matches = matchRes.subgroups
                res = Option(new VideoData(
                    videoItem.id,
                    videoItem.timestamp,
                    fixGame(matches(0)),
                    fixPlayers(matches(1), matches(3)),
                    fixCharacters(matches(2), matches(4))
                ))
            }
            case None => {}
        }
        res
    }
}

object VideoManager {
    private val DATA_FILE_PATH = "data/video.json"
    private val parsers = List(
        YogaFlameParser,
        OlympicGamingParser
    )
<commit_msg>generisize the parsers with inheritance<commit_after>
package fgc.formatter

import fgc.scraper.VideoItem
import fgc.scraper.YouTubeChannel

import java.io._
import scala.util.matching.Regex

import scalaj.http._
import org.json4s._
import org.json4s.native.Serialization

case class VideoData(
    id: String,
    timestamp: String,
    game: String,
    players: List[String],
    characters: List[List[String]]
){
    val tuple: List[Any] = List(
        timestamp, id, game, players, characters
    )
}

trait ChannelParser {
    val channel: YouTubeChannel
    def loadVideos(): List[VideoData] = {
        channel.loadFile.values.flatMap(parseVideo).toList
    }

    trait VideoParser {
        val regex: Regex
        def parseSuccess(videoItem: VideoItem, regexMatch: List[String]): VideoData {}
    }
    val parsers: List[VideoParser]
    def parseVideo(videoItem: VideoItem): Option[VideoData] = {
        var toRet: Option[VideoData] = None
        parsers.foreach { parser =>
            val matchMaybe = parser.regex.findFirstMatchIn(videoItem.title)
            matchMaybe match {
                case Some(matchRes) => {
                    val regexMatch = matchRes.subgroups
                    toRet = Option(parser.parseSuccess(videoItem, regexMatch))
                }
                case None => {}
            }
        }
        if (toRet == None){
            // failed regex, maybe log?
        }
        toRet
    }

    val rPlayer = "([\\w\\.\\-\\| ]+) "
    val rCharacter = "(?:\\(|\\[) *([\\w\\. ]+) *(?:\\)|\\]) "
    val rVersus = "(?:Vs|vs)\\.? "
    val rGameMap: Map[String, String]
    def rGame(): String = {
        val rGames = rGameMap.values.mkString("|")
        s" *($rGames) *"
    }

    def fixGame(rawGame: String): String = {
        // todo improve this
        var matchingKey = ""
        rGameMap.foreach { case (key, value) =>
            val matchMaybe = value.r.findFirstIn(rawGame)
            matchMaybe match {
                case Some(name) =>
                    matchingKey = key
                case None =>
                    // do nothing
            }
        }
        matchingKey
    }
    def fixCharacters(char1: String, char2: String): List[List[String]] = {
        List(List(char1), List(char2))
    }
    def fixPlayers(player1: String, player2: String): List[String] = {
        List(player1, player2)
    }
}

object YogaFlameParser extends ChannelParser {
    val channel = YouTubeChannel.YogaFlame

    val rGameMap = Map(
        "SF5" -> "SF5|SFV|Beta SFV",
        "USF4" -> "USF4",
        "SSF4AE2012" -> "(?:Arcade Edition|AE)(?: Version)? +2012",
        "SSF4AE" -> "Arcade Edition|AE",
        "SF3" -> "SF3",
        "SFxT" -> "SFxT",
        "TTT2" -> "Tekken Tag Tournament 2"
    )
    val rRounds = "(?:X[0-9] )?"
    val rResolution = " *(?:1080p|720p)"

    object GameLastParser extends VideoParser {
        val regex = (
            rRounds +
            rPlayer +
            rCharacter +
            rVersus +
            rPlayer +
            rCharacter +
            rGame +
            rResolution
        ).r
        def parseSuccess(videoItem: VideoItem, regexMatch: List[String]): VideoData = {
            new VideoData(
                videoItem.id,
                videoItem.timestamp,
                fixGame(regexMatch(4)),
                fixPlayers(regexMatch(0), regexMatch(2)),
                fixCharacters(regexMatch(1), regexMatch(3))
            )
        }
    }
    val parsers = List(GameLastParser)
}

object OlympicGamingParser extends ChannelParser {
    val channel = YouTubeChannel.OlympicGaming

    val rGameMap = Map(
        "SF5" -> "Street Fighter (?:5 *\\/? *V|5|V)|SFV",
        "SFxT" -> "Street Fighter X Tekken",
        "P4AU" -> "Persona 4 Arena Ultimax",
        "MKX" -> "Mortal Kombat X",
        "SC5" -> "SoulCalibur 5\\/V",
        "DOA5" -> "Dead Or Alive 5 Last Round",
        "Smash4" -> "Super Smash Bros Wii U",
        "GGXrd" -> "Guilty Gear Xrd",
        "KI" -> "Killer Instinct"
    )
    // val rEndtag = " *(?:[\\w ]*-? *Gameplay).*"
    val rEndtag = "(?:Wii|Xbox|PS4|-? ?Gameplay).*"

    object GameLastParser extends VideoParser {
        val regex = (
            rPlayer +
            rCharacter +
            rVersus +
            rPlayer +
            rCharacter +
            rGame +
            rEndtag
        ).r
        def parseSuccess(videoItem: VideoItem, regexMatch: List[String]): VideoData = {
            new VideoData(
                videoItem.id,
                videoItem.timestamp,
                fixGame(regexMatch(4)),
                fixPlayers(regexMatch(0), regexMatch(2)),
                fixCharacters(regexMatch(1), regexMatch(3))
            )
        }
    }
    object GameFirstParser extends VideoParser {
        val regex = (
            rGame + ": " +
            rPlayer +
            rCharacter +
            rVersus +
            rPlayer +
            rCharacter +
            rEndtag
        ).r
        def parseSuccess(videoItem: VideoItem, regexMatch: List[String]): VideoData = {
            new VideoData(
                videoItem.id,
                videoItem.timestamp,
                fixGame(regexMatch(0)),
                fixPlayers(regexMatch(1), regexMatch(3)),
                fixCharacters(regexMatch(2), regexMatch(4))
            )
        }
    }
    val parsers = List(GameLastParser, GameFirstParser)
}

object VideoManager {
    private val DATA_FILE_PATH = "data/video.json"
    private val parsers = List(
        YogaFlameParser,
        OlympicGamingParser
    )

{'fraction_non_alphanumeric': 0.12306985294117648, 'fraction_numerical': 0.012316176470588235, 'mean_word_length': 1.6435860058309038, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1511041', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3236, 'n_words': 709}