<commit_before><commit_msg>test ip speed<commit_after>require 'socket'
require 'thread'
require 'timeout'



def ip_speed_test(dt,&block)
  Thread.new do
    # http://releases.ubuntu.com/releases/14.04/ubuntu-14.04-desktop-i386.iso
    iso="ubuntu-14.04-desktop-i386.iso"
    host="releases.ubuntu.com"
    uri="http://#{host}"
    path="/releases/14.04"
    url="#{uri}#{path}/#{iso}"

    header= <<EOF
GET #{url} HTTP/1.0
User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:30.0) Gecko/20100101 Firefox/30.0
Host: iso.linuxquestions.org
Connection: close

EOF
    header.gsub!(/\n/,"\r\n")
    puts header
	  timeout(12000) {
		  s = TCPSocket.open(host, 80) 
      s.sync=true
      s.print header
      content=s.recv(1)
		  ss=0
      start=Time.now.to_f
		  loop  do
        content=s.recv(1024)
        break unless content && content.size>0
			  ss+=content.size
        now=Time.now.to_f
        if (now-start)>dt
          speed=ss/(1024*(now-start))
          if block
           block.call(ss,now-start,speed) 
          else
            puts "size #{ss} during #{now-start} seconds >> speed= #{speed} KB/s"
          end
				  ss=0
				  start=now
			  end
		  end
		  s.close
	  } rescue puts "Error #{$!}"
    if block_given?
      yield(0,0,0) 
    end
  end 
end


if $0==__FILE__
  th=ip_speed_test(1) { |qt,delta,speed| puts "speed #{speed.round} KB/s" }
  sleep 30
  puts "kill thread..."
  th.kill
end

{'fraction_non_alphanumeric': 0.15368421052631578, 'fraction_numerical': 0.04982456140350877, 'mean_word_length': 2.355294117647059, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '572069', 'n_tokens_mistral': 619, 'n_tokens_neox': 527, 'n_words': 105}