<commit_before>require 'ipaddr'
require 'logger'

module Dumper
  module Utility
    class IP
      attr_reader :ip, :ipaddr

      def initialize(*args)
        disable_reverse_lookup do
          UDPSocket.open do |s|
            s.connect '172.16.58.3', 1
            @ip = s.addr.last
          end
        end
        @ipaddr = Dumper::Utility::IPAddr.new(@ip)
      end

      def disable_reverse_lookup
        orig = Socket.do_not_reverse_lookup
        Socket.do_not_reverse_lookup = true
        begin
          yield
        ensure
          Socket.do_not_reverse_lookup = orig
        end
      end
    end

    class IPAddr < ::IPAddr
      def private?
        return false unless self.ipv4?

        [ IPAddr.new("10.0.0.0/8"),
          IPAddr.new("172.16.0.0/12"),
          IPAddr.new("192.168.0.0/16") ].any?{|i| i.include? self }
      end
    end

    class SlimLogger < Logger
      def initialize(logdev, shift_age = 0, shift_size = 1048576)
        super
        self.formatter = SlimFormatter.new
        self.formatter.datetime_format = "%Y-%m-%dT%H:%M:%S"
        self.level = Logger::INFO
      end
      
      class SlimFormatter < Logger::Formatter
        # This method is invoked when a log event occurs
        def call(severity, time, progname, msg)
          "[%s] %5s : %s\n" % [format_datetime(time), severity, msg2str(msg)]<commit_msg>Adding pid to the log.<commit_after>require 'ipaddr'
require 'logger'

module Dumper
  module Utility
    class IP
      attr_reader :ip, :ipaddr

      def initialize(*args)
        disable_reverse_lookup do
          UDPSocket.open do |s|
            s.connect '172.16.58.3', 1
            @ip = s.addr.last
          end
        end
        @ipaddr = Dumper::Utility::IPAddr.new(@ip)
      end

      def disable_reverse_lookup
        orig = Socket.do_not_reverse_lookup
        Socket.do_not_reverse_lookup = true
        begin
          yield
        ensure
          Socket.do_not_reverse_lookup = orig
        end
      end
    end

    class IPAddr < ::IPAddr
      def private?
        return false unless self.ipv4?

        [ IPAddr.new("10.0.0.0/8"),
          IPAddr.new("172.16.0.0/12"),
          IPAddr.new("192.168.0.0/16") ].any?{|i| i.include? self }
      end
    end

    class SlimLogger < Logger
      def initialize(logdev, shift_age = 0, shift_size = 1048576)
        super
        self.formatter = SlimFormatter.new
        self.formatter.datetime_format = "%Y-%m-%dT%H:%M:%S"
        self.level = Logger::INFO
      end
      
      class SlimFormatter < Logger::Formatter
        # This method is invoked when a log event occurs
        def call(severity, time, progname, msg)
          "[%s (%d)] %5s : %s\n" % [format_datetime(time), $$, severity, msg2str(msg)]
{'fraction_non_alphanumeric': 0.10665220535068691, 'fraction_numerical': 0.03253796095444685, 'mean_word_length': 1.906512605042017, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5733269', 'n_tokens_mistral': 944, 'n_tokens_neox': 874, 'n_words': 206}