<commit_before>require 'csv'

module Cranson
  module Parsers
    class Dcc
      include Observable

      def parse(file)
        each_row(file) do |row|
          @current_object = build_planning_application(row)
        end
      end

      private

      def notify
        changed
        notify_observers(@current_object)
      end

      def each_row(file)
        CSV.foreach(file, headers: true) { |row| yield row }
      end

      def build_planning_application(data)
        Cranson::PlanningApplication.new.tap do |plan|
          plan.planning_reference = data["Plan_Ref"].strip
          plan.description = data["PROPOSAL"]<commit_msg>fixes encoding error by specifying the expected format<commit_after>require 'csv'

module Cranson
  module Parsers
    class Dcc
      include Observable

      def parse(file)
        each_row(file) do |row|
          @current_object = build_planning_application(row)
        end
      end

      private

      def notify
        changed
        notify_observers(@current_object)
      end

      def each_row(file)
        CSV.foreach(file, headers: true, encoding: 'windows-1251:utf-8') { |row| yield row }
      end

      def build_planning_application(data)
        Cranson::PlanningApplication.new.tap do |plan|
          plan.planning_reference = data["Plan_Ref"].strip
          plan.description = data["PROPOSAL"]
{'fraction_non_alphanumeric': 0.07781818181818181, 'fraction_numerical': 0.0036363636363636364, 'mean_word_length': 2.3891625615763545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5078878', 'n_tokens_mistral': 404, 'n_tokens_neox': 390, 'n_words': 94}