<commit_before># encoding: UTF-8
require 'date'
require 'influxdb'

class Fluent::InfluxdbOutput < Fluent::BufferedOutput
  Fluent::Plugin.register_output('influxdb', self)

  config_param :host, :string,  :default => 'localhost'
  config_param :port, :integer,  :default => 8086
  config_param :dbname, :string,  :default => 'fluentd'
  config_param :user, :string,  :default => 'root'
  config_param :password, :string,  :default => '<PASSWORD>'
  config_param :time_precision, :string, :default => 's'
  config_param :use_ssl, :bool, :default => false


  def initialize
    super
  end

  def configure(conf)
    super
    @influxdb = InfluxDB::Client.new @dbname, host: @host, 
                                              port: @port, 
                                              username: @user, 
                                              password: <PASSWORD>,
                                              async: false,
                                              time_precision: @time_precision,
                                              use_ssl: @use_ssl

  end

  def start
    super
  end

  def format(tag, time, record)
    [tag, time, record].to_msgpack
  end

  def shutdown
    super
  end

  def write(chunk)
    chunk.msgpack_each do |tag, time, record|
      unless record.empty?
        record[:time] = time
        @influxdb.write_point(tag, record)
      end
    end
  end
end
<commit_msg>added the ability to remove a tag prefix<commit_after># encoding: UTF-8
require 'date'
require 'influxdb'

class Fluent::InfluxdbOutput < Fluent::BufferedOutput
  Fluent::Plugin.register_output('influxdb', self)

  config_param :host, :string,  :default => 'localhost'
  config_param :port, :integer,  :default => 8086
  config_param :dbname, :string,  :default => 'fluentd'
  config_param :user, :string,  :default => 'root'
  config_param :password, :string,  :default => '<PASSWORD>'
  config_param :time_precision, :string, :default => 's'
  config_param :use_ssl, :bool, :default => false
  config_param :remove_tag_prefix, :string, :default => ''


  def initialize
    super
  end

  def configure(conf)
    super
    @remove_tag_prefix = conf['remove_tag_prefix']
    @influxdb = InfluxDB::Client.new @dbname, host: @host,
                                              port: @port,
                                              username: @user,
                                              password: <PASSWORD>,
                                              async: false,
                                              time_precision: @time_precision,
                                              use_ssl: @use_ssl

  end

  def start
    super
  end

  def format(tag, time, record)
    tag.gsub!(/^#{@remove_tag_prefix}(\.)?/, '') unless @remove_tag_prefix.empty?
    [tag, time, record].to_msgpack
  end



  def shutdown
    super
  end

  def write(chunk)
    chunk.msgpack_each do |tag, time, record|
      unless record.empty?
        record[:time] = time
        @influxdb.write_point(tag, record)
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.10332464146023468, 'fraction_numerical': 0.003259452411994785, 'mean_word_length': 1.8735955056179776, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4917863', 'n_tokens_mistral': 898, 'n_tokens_neox': 855, 'n_words': 226}