<commit_before>class Fluentd
  module Setting
    class InTail
      include ActiveModel::Model
      attr_accessor :path, :tag, :format, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval

      validates :path, presence: true
      validates :tag, presence: true
      #validates :format, presence: true

      def self.known_formats
        {
          :regexp => [],
          :apache2 => [:time_format],
          :nginx => [:time_format],
          :syslog => [:time_format],
          :tsv => [:keys, :time_key],
          :csv => [:keys, :time_key],
          :ltsv => [:delimiter, :time_key],
          :json => [:time_key],
          :multiline => [:format_firstline, *(1..20).map{|n| "format#{n}".to_sym}],
        }
      end
      attr_accessor *known_formats.values.flatten.compact

      def known_formats
        self.class.known_formats
      end

      def guess_format
        case path
        when /\.json$/
          :json
        when /\.csv$/
          :csv
        when /\.tsv$/
          :tsv
        when /\.ltsv$/
          :ltsv
        when /nginx/
          :nginx
        when /apache/
          :apache2
        when %r|/var/log|
          :syslog
        else
          :regexp
        end
      end

      def to_conf
        <<-XML.strip_heredoc.gsub(/^[ ]+\n/m, "")
          <source>
            type tail
            path #{path}
            tag #{tag}
            #{read_from_head.to_i.zero? ? "" : "read_from_head true"}
            #{pos_file.present? ? "pos_file #{pos_file}" : ""}
            #{rotate_wait.present? ? "rotate_wait #{rotate_wait}" : ""}
            #{refresh_interval.present? ? "refresh_interval #{refresh_interval}" : ""}
          </source>
        XML
      end
    end
  end
end
<commit_msg>Remove multiline support<commit_after>class Fluentd
  module Setting
    class InTail
      include ActiveModel::Model
      attr_accessor :path, :tag, :format, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval

      validates :path, presence: true
      validates :tag, presence: true
      #validates :format, presence: true

      def self.known_formats
        {
          :regexp => [],
          :apache2 => [:time_format],
          :nginx => [:time_format],
          :syslog => [:time_format],
          :tsv => [:keys, :time_key],
          :csv => [:keys, :time_key],
          :ltsv => [:delimiter, :time_key],
          :json => [:time_key],
        }
      end
      attr_accessor *known_formats.values.flatten.compact

      def known_formats
        self.class.known_formats
      end

      def guess_format
        case path
        when /\.json$/
          :json
        when /\.csv$/
          :csv
        when /\.tsv$/
          :tsv
        when /\.ltsv$/
          :ltsv
        when /nginx/
          :nginx
        when /apache/
          :apache2
        when %r|/var/log|
          :syslog
        else
          :regexp
        end
      end

      def to_conf
        <<-XML.strip_heredoc.gsub(/^[ ]+\n/m, "")
          <source>
            type tail
            path #{path}
            tag #{tag}
            #{read_from_head.to_i.zero? ? "" : "read_from_head true"}
            #{pos_file.present? ? "pos_file #{pos_file}" : ""}
            #{rotate_wait.present? ? "rotate_wait #{rotate_wait}" : ""}
            #{refresh_interval.present? ? "refresh_interval #{refresh_interval}" : ""}
          </source>
        XML
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.13256814921090387, 'fraction_numerical': 0.0020086083213773313, 'mean_word_length': 1.586053412462908, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5804885', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1107, 'n_words': 234}