<commit_before># this is a beautification wrapper around the low-level
# MT940.parse command. use it in order to make dealing with 
# the data easier
class MT940

  class CustomerStatementMessage

    attr_reader :statement_lines

    def self.parse_file(file)
      string = File.read(file)
      messages = MT940.parse(string)
      messages.map { |msg| new(msg) }
    end
    
    def initialize(raw_mt940)
      @raw = raw_mt940
      @account = @raw.find { |line| line.class == MT940::Account }
      @statement_lines = []
      @raw.each_with_index do |line, i|
        next unless line.class == MT940::StatementLine
        ensure_is_info_line!(@raw[i+1])
        @statement_lines << StatementLineBundle.new(@raw[i], @raw[i+1])
      end
    end

    def bank_code
      @account.bank_code
    end

    def account_number
      @account.account_number
    end

    private 
    
    def ensure_is_info_line!(line)
      unless line.is_a?(MT940::StatementLineInformation)
        raise StandardError, "Unexpected Structure; expected StatementLineInformation, but was #{line.class}" 
      end
    end
    
  end
  
  class StatementLineBundle

    METHOD_MAP = {
      :amount         => :line,
      :funds_code     => :line,
      :account_holder => :info,
      :details        => :info,
      :account_number => :info,
      :bank_code      => :info
    }

    def initialize(statement_line, statement_line_info)
      @line, @info = statement_line, statement_line_info
    end

    def method_missing(method, *args, &block)
      super unless METHOD_MAP.has_key?(method)
      object = instance_variable_get("@#{METHOD_MAP[method.to_sym]}")
      object.send(method)
    end

  end
  
end<commit_msg>Update offline donation statement import to handle file encodings.<commit_after># this is a beautification wrapper around the low-level
# MT940.parse command. use it in order to make dealing with 
# the data easier
class MT940

  class CustomerStatementMessage

    attr_reader :statement_lines

    def self.parse_file(file)
      self.parse(File.read(file))
    end
    
    def self.parse(data)
      messages = MT940.parse(data)
      messages.map { |msg| new(msg) }
    end
    
    def initialize(raw_mt940)
      @raw = raw_mt940
      @account = @raw.find { |line| line.class == MT940::Account }
      @statement_lines = []
      @raw.each_with_index do |line, i|
        next unless line.class == MT940::StatementLine
        ensure_is_info_line!(@raw[i+1])
        @statement_lines << StatementLineBundle.new(@raw[i], @raw[i+1])
      end
    end

    def bank_code
      @account.bank_code
    end

    def account_number
      @account.account_number
    end

    private 
    
    def ensure_is_info_line!(line)
      unless line.is_a?(MT940::StatementLineInformation)
        raise StandardError, "Unexpected Structure; expected StatementLineInformation, but was #{line.class}" 
      end
    end
    
  end
  
  class StatementLineBundle

    METHOD_MAP = {
      :amount         => :line,
      :funds_code     => :line,
      :account_holder => :info,
      :details        => :info,
      :account_number => :info,
      :bank_code      => :info
    }

    def initialize(statement_line, statement_line_info)
      @line, @info = statement_line, statement_line_info
    end

    def method_missing(method, *args, &block)
      super unless METHOD_MAP.has_key?(method)
      object = instance_variable_get("@#{METHOD_MAP[method.to_sym]}")
      object.send(method)
    end

  end
  
end
{'fraction_non_alphanumeric': 0.09675583380762663, 'fraction_numerical': 0.014797951052931132, 'mean_word_length': 2.646265560165975, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2874946', 'n_tokens_mistral': 1137, 'n_tokens_neox': 1074, 'n_words': 255}