<commit_before>require "logstash/codecs/base"

class LogStash::Codecs::OldLogStashJSON < LogStash::Codecs::Base
  config_name "oldlogstashjson"
  milestone 1

  public
  def decode(data)
    obj = JSON.parse(data.force_encoding("UTF-8"))

    h  = {}

    # Convert the old logstash schema to the new one.
    basics = %w(@timestamp @message @source_host @source_path @source
                @tags @type)
    basics.each do |key|
      h[key] = obj[key] if obj.include?(key)
    end

    h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
    event = LogStash::Event.new(h)
  end # def decode

  public
  def encode(data)
    h  = {}

    h["@timestamp"] = data["@timestamp"]
    h["@message"] = data["message"] if data.include?("message")
    h["@source_host"] = data["source_host"] if data.include?("source_host")
    # Convert the old logstash schema to the new one.
    basics = %w(@timestamp @message @source_host @source_path @source
                @tags @type)
    basics.each do |key|
      h[key] = obj[key] if obj.include?(key)
    end

    h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
    @on_event.call(h)
  end # def encode

end # class LogStash::Codecs::Json<commit_msg>- fix bug: actually yield events on decode - convert @foo names to just 'foo'<commit_after>require "logstash/codecs/base"

class LogStash::Codecs::OldLogStashJSON < LogStash::Codecs::Base
  config_name "oldlogstashjson"
  milestone 1

  public
  def decode(data)
    obj = JSON.parse(data.force_encoding("UTF-8"))

    h  = {}

    # Convert the old logstash schema to the new one.
    basics = %w(@timestamp @message @source_host @source_path @source
                @tags @type)
    basics.each do |key|
      # Convert '@message' to 'message', etc
      h[key[1..-1]] = obj[key] if obj.include?(key)
    end

    h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
    yield LogStash::Event.new(h)
  end # def decode

  public
  def encode(data)
    h  = {}

    h["@timestamp"] = data["@timestamp"]
    h["@message"] = data["message"] if data.include?("message")
    h["@source_host"] = data["source_host"] if data.include?("source_host")
    # Convert the old logstash schema to the new one.
    basics = %w(@timestamp @message @source_host @source_path @source
                @tags @type)
    basics.each do |key|
      h[key] = obj[key] if obj.include?(key)
    end

    h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
    @on_event.call(h)
  end # def encode

end # class LogStash::Codecs::Json
{'fraction_non_alphanumeric': 0.1532643312101911, 'fraction_numerical': 0.0023885350318471337, 'mean_word_length': 3.2957264957264956, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6423997', 'n_tokens_mistral': 881, 'n_tokens_neox': 867, 'n_words': 211}