<commit_before>#!/usr/bin/env ruby

require 'yaml'
require 'json'
require 'csv'

module AdHocTemplate
  module RecordReader
    module YAMLReader
      def self.read_record(yaml_data)
        RecordReader.convert_values_to_string(YAML.load(yaml_data))
      end

      def self.dump(config_data)
        data = RecordReader.parse_if_necessary(config_data)
        YAML.dump(data)
      end
    end

    module JSONReader
      def self.read_record(json_data)
        RecordReader.convert_values_to_string(JSON.parse(json_data))
      end

      def self.dump(config_data)
        data = RecordReader.parse_if_necessary(config_data)
        JSON.pretty_generate(data)
      end
    end

    module CSVReader
      COL_SEP = {
        csv: CSV::DEFAULT_OPTIONS[:col_sep],
        tsv: "\t",
      }

      module HEADER_POSITION
        TOP = '__header_top__'
        LEFT = '__header_left__'
      end

      class NotSupportedError < StandardError; end

      def self.read_record(csv_data, config={ csv: nil })
        label, sep = parse_config(config)
        header, *data = csv_to_array(csv_data, sep, label)
        csv_records = data.map {|row| convert_to_hash(header, row) }
        if label && label.index('|')
          return compose_inner_iteration_records(csv_records, label)
        end
        compose_record(csv_records, label)
      end

      def self.dump(config_data, col_sep=COL_SEP[:csv])
        data = RecordReader.parse_if_necessary(config_data)
        raise NotSupportedError unless csv_compatible_format?(data)

        if kv_pairs = find_sub_records(data)
          records = hashes_to_arrays(kv_pairs)
        else
          records = data.to_a.transpose
        end

        array_to_csv(records, col_sep)
      end

      def self.convert_to_hash(header, row_array)
        {}.tap do |record|
          header.zip(row_array).each {|key, value| record[key] = value }
        end
        # if RUBY_VERSION >= 2.1.0: header.zip(row_array).to_h
      end<commit_msg>refactoring of CSVReader.dump<commit_after>#!/usr/bin/env ruby

require 'yaml'
require 'json'
require 'csv'

module AdHocTemplate
  module RecordReader
    module YAMLReader
      def self.read_record(yaml_data)
        RecordReader.convert_values_to_string(YAML.load(yaml_data))
      end

      def self.dump(config_data)
        data = RecordReader.parse_if_necessary(config_data)
        YAML.dump(data)
      end
    end

    module JSONReader
      def self.read_record(json_data)
        RecordReader.convert_values_to_string(JSON.parse(json_data))
      end

      def self.dump(config_data)
        data = RecordReader.parse_if_necessary(config_data)
        JSON.pretty_generate(data)
      end
    end

    module CSVReader
      COL_SEP = {
        csv: CSV::DEFAULT_OPTIONS[:col_sep],
        tsv: "\t",
      }

      module HEADER_POSITION
        TOP = '__header_top__'
        LEFT = '__header_left__'
      end

      class NotSupportedError < StandardError; end

      def self.read_record(csv_data, config={ csv: nil })
        label, sep = parse_config(config)
        header, *data = csv_to_array(csv_data, sep, label)
        csv_records = data.map {|row| convert_to_hash(header, row) }
        if label && label.index('|')
          return compose_inner_iteration_records(csv_records, label)
        end
        compose_record(csv_records, label)
      end

      def self.dump(config_data, col_sep=COL_SEP[:csv])
        data = RecordReader.parse_if_necessary(config_data)
        raise NotSupportedError unless csv_compatible_format?(data)

        kv_pairs = find_sub_records(data)
        records = kv_pairs ? hashes_to_arrays(kv_pairs) : data.to_a.transpose

        array_to_csv(records, col_sep)
      end

      def self.convert_to_hash(header, row_array)
        {}.tap do |record|
          header.zip(row_array).each {|key, value| record[key] = value }
        end
        # if RUBY_VERSION >= 2.1.0: header.zip(row_array).to_h
      end
{'fraction_non_alphanumeric': 0.08877679697351828, 'fraction_numerical': 0.0015132408575031526, 'mean_word_length': 2.6054545454545455, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3087718', 'n_tokens_mistral': 1320, 'n_tokens_neox': 1258, 'n_words': 242}