<commit_before>require 'fileutils'
require 'csv'

module KeplerProcessor
  class Base

    def initialize(input_filename, options)
      @input_filename   = input_filename
      @options          = options
      @input_data       = []
    end

    def run
      puts "Processing file #{@input_filename}"
      read_in_data
      split_comments!
      parse_header_attributes
      convert_from_string!
      yield
      save!
      puts "Finished processing file #{@input_filename}"
    end

    def full_output_filename
      "#{@options[:output_path]}/#{output_filename}"
    end

    private

      def read_in_data
        File.new(@input_filename, "r").each { |line| @input_data << line }
        # file closes automatically because it's opened in this method
        raise NoDataError if @input_data.empty?
      end

      def split_comments!
        # matches (=~) regular expression (/../) hash at start of line (^), preceeded by any number of spaces (\s*)
        @comments, @input_data = @input_data.partition { |line| line =~ /^(\s*#)/ }
      end

      def parse_header_attributes
        # select lines from comments containing a colon, map them into an array, remove the '#' and split
        # about that colon. Create a hash out of the result.
        @attributes = @comments.select { |line| line.include? ":" }.map { |line| line.gsub("# ", "").split ":" }.to_hash
      end

      def convert_from_string!
        # convert @input_data to a two dimensional float array: time, flux
        # @input_data.map! { |line| line.split(" ").map(&:to_f)[ 0..1 ] }
        @input_data.map! do |line|
          l = line.split(" ").map &:to_f
          [l[0], l[1]]
        end
      end

      def output_filename
        nil # defaults to nil, child class must override output_filename in order to save
      end

      def save!
        if output_filename
          @output_data ||= @input_data
          ::FileUtils.mkpath @options[:output_path]
          raise FileExistsError if File.exist?(full_output_filename) && !@options[:force_overwrite]
          puts "Writing output to #{full_output_filename}"
          CSV.open(full_output_filename, "a+", :col_sep => "\t") do |csv| # 'a+' for all - read, write... everything<commit_msg>remove old code. column most commonly used is 3rd.<commit_after>require 'fileutils'
require 'csv'

module KeplerProcessor
  class Base

    def initialize(input_filename, options)
      @input_filename   = input_filename
      @options          = options
      @input_data       = []
    end

    def run
      puts "Processing file #{@input_filename}"
      read_in_data
      split_comments!
      parse_header_attributes
      convert_from_string!
      yield
      save!
      puts "Finished processing file #{@input_filename}"
    end

    def full_output_filename
      "#{@options[:output_path]}/#{output_filename}"
    end

    private

      def read_in_data
        File.new(@input_filename, "r").each { |line| @input_data << line }
        # file closes automatically because it's opened in this method
        raise NoDataError if @input_data.empty?
      end

      def split_comments!
        # matches (=~) regular expression (/../) hash at start of line (^), preceeded by any number of spaces (\s*)
        @comments, @input_data = @input_data.partition { |line| line =~ /^(\s*#)/ }
      end

      def parse_header_attributes
        # select lines from comments containing a colon, map them into an array, remove the '#' and split
        # about that colon. Create a hash out of the result.
        @attributes = @comments.select { |line| line.include? ":" }.map { |line| line.gsub("# ", "").split ":" }.to_hash
      end

      def convert_from_string!
        # convert @input_data to a two dimensional float array: time, flux
        @input_data.map! do |line|
          l = line.split(" ").map &:to_f
          [l[0], l[3]]
        end
      end

      def output_filename
        nil # defaults to nil, child class must override output_filename in order to save
      end

      def save!
        if output_filename
          @output_data ||= @input_data
          ::FileUtils.mkpath @options[:output_path]
          raise FileExistsError if File.exist?(full_output_filename) && !@options[:force_overwrite]
          puts "Writing output to #{full_output_filename}"
          CSV.open(full_output_filename, "a+", :col_sep => "\t") do |csv| # 'a+' for all - read, write... everything
{'fraction_non_alphanumeric': 0.10829027184902269, 'fraction_numerical': 0.0015726802965625702, 'mean_word_length': 2.464591439688716, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7426937', 'n_tokens_mistral': 1305, 'n_tokens_neox': 1278, 'n_words': 396}