<commit_before># encoding: utf-8

module SportDb

class MatchReader    ## todo/check: rename to MatchReaderV2 (use plural?) why? why not?

  def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
    parse( txt, season: season )
  end

  def self.parse( txt, season: nil )
    new( txt ).parse( season: season )
  end


  include Logging

  def initialize( txt )
    @txt = txt
  end

  def parse( season: nil )
    secs = LeagueOutlineReader.parse( @txt, season: season )
    pp secs


    ###
    ## todo/check/fix:  move to LeagueOutlineReader for (re)use - why? why not?
    ##                    use sec[:lang] or something?
    langs = {    ## map country keys to lang codes
      'de' => 'de', ## de - Deutsch (German)
      'at' => 'de',
      'fr' => 'fr', ## fr - French
      'it' => 'it', ## it - Italian
      'es' => 'es', ## es - Español (Spanish)
      'mx' => 'es',
      'pt' => 'pt', ## pt -  Português (Portuguese)
      'br' => 'br'
    }

    secs.each do |sec|   ## sec(tion)s
      season = sec[:season]
      league = sec[:league]
      stage  = sec[:stage]
      lines  = sec[:lines]

      ## hack for now: switch lang
      ## todo/fix: set lang for now depending on league country!!!
      if league.intl?   ## todo/fix: add intl? to ActiveRecord league!!!
        Import.config.lang = 'en'
      else  ## assume national/domestic
        Import.config.lang = langs[ league.country.key ] || 'en'
      end


      start = if season.year?
                Date.new( season.start_year, 1, 1 )
              else
                Date.new( season.start_year, 7, 1 )
              end

      auto_conf_teams, _ = AutoConfParser.parse( lines,
                                                 start: start )

      ## step 1: map/find teams

      ## note: loop over keys (holding the names); values hold the usage counter!! e.g. 'Arsenal' => 2, etc.<commit_msg>up reader<commit_after># encoding: utf-8

module SportDb

class MatchReader    ## todo/check: rename to MatchReaderV2 (use plural?) why? why not?

  def self.read( path, season: nil )   ## use - rename to read_file or from_file etc. - why? why not?
    txt = File.open( path, 'r:utf-8' ) {|f| f.read }
    parse( txt, season: season )
  end

  def self.parse( txt, season: nil )
    new( txt ).parse( season: season )
  end


  include Logging

  def initialize( txt )
    @txt = txt
  end

  def parse( season: nil )
    secs = LeagueOutlineReader.parse( @txt, season: season )
    pp secs


    ###
    ## todo/check/fix:  move to LeagueOutlineReader for (re)use - why? why not?
    ##                    use sec[:lang] or something?
    langs = {    ## map country keys to lang codes
      'de' => 'de', ## de - Deutsch (German)
      'at' => 'de',
      'fr' => 'fr', ## fr - French
      'it' => 'it', ## it - Italian
      'es' => 'es', ## es - Español (Spanish)
      'mx' => 'es',
      'pt' => 'pt', ## pt -  Português (Portuguese)
      'br' => 'br'
    }

    secs.each do |sec|   ## sec(tion)s
      season = sec[:season]
      league = sec[:league]
      stage  = sec[:stage]
      lines  = sec[:lines]

      ## hack for now: switch lang
      ## todo/fix: set lang for now depending on league country!!!
      if league.intl?   ## todo/fix: add intl? to ActiveRecord league!!!
        Import.config.lang = 'en'
      else  ## assume national/domestic
        Import.config.lang = langs[ league.country.key ] || 'en'
      end

      ### check if event info availabe - use start_date;
      ##    otherwise we have to guess (use a "synthetic" start_date)
      event_info = catalog.events.find_by( season: season,
                                           league: league )

      start = if event_info && event_info.start_date
                  event_info.start_date
              else
                if season.year?
                  Date.new( season.start_year, 1, 1 )
                else
                  Date.new( season.start_year, 7, 1 )
                end
              end

      auto_conf_teams, _ = AutoConfParser.parse( lines,
                                                 start: start )

      ## step 1: map/find teams

      ## note: loop over keys (holding the names); values hold the usage counter!! e.g. 'Arsenal' => 2, etc.
{'fraction_non_alphanumeric': 0.13144567219152856, 'fraction_numerical': 0.004143646408839779, 'mean_word_length': 1.8491803278688526, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1883298', 'n_tokens_mistral': 1404, 'n_tokens_neox': 1331, 'n_words': 434}