<commit_before>#!/usr/bin/env ruby
#
#
require 'thread'
require 'socket'
require File.expand_path("../thin_http_parser", __FILE__)

class SingletonQueue

  def self.new(*args)
    # Guarantees that only a single instance will exist
    @@instance ||= super(*args)
  end

  def self.get
    # Helper to make what's going on more obvious
    @@instance
  end

######

  def initialize(opts={})
    @in_q  = Queue.new
    @out_q = Queue.new
    opts[:port] ||= (ENV["BACKEND_PORT"] || 2000).to_i
    spawn_reader(opts[:port])
  end
  attr_reader :in_q, :out_q

  def enq(data)
    in_q.enq(data)
  end

  def deq
    out_q.deq
  end

  def join!
    @thread.join
  end

private

  def spawn_reader(port)
    @thread = Thread.new(in_q, out_q) do |en_q, de_q|
      serv = TCPServer.new port
      loop do
        # Create a parser object
        parser = RequestParser.new

        # Pop a connection off the stack
        client = serv.accept

        # retrieve the headers
        lines = []
        while (line = client.gets) != "\r\n"
          # XXX Hack around missing CONTENT_LENGTH in parser.env
          if parser.env['CONTENT_LENGTH'].nil? && line =~ /^Content-Length: (\d+)/
            parser.env['CONTENT_LENGTH'] = $1.to_i
          end
          lines << line
        end

        parser.parse(lines.join("\r\n"))

        if parser.content_length > 0
          parser.parse(client.gets(parser.content_length))
        end

        # Send the request object back to the caller
        de_q.enq(parser)

        # Fetch the response object from the caller
        response = en_q.deq
        client.write(response.to_s)

        client.close
      end
    end
  end

end
<commit_msg>Implement our own header parser<commit_after>#!/usr/bin/env ruby
#
#
require 'thread'
require 'socket'
require File.expand_path("../thin_http_parser", __FILE__)

class SingletonQueue

  def self.new(*args)
    # Guarantees that only a single instance will exist
    @@instance ||= super(*args)
  end

  def self.get
    # Helper to make what's going on more obvious
    @@instance
  end

######

  def initialize(opts={})
    @in_q  = Queue.new
    @out_q = Queue.new
    opts[:port] ||= (ENV["BACKEND_PORT"] || 2000).to_i
    spawn_reader(opts[:port])
  end
  attr_reader :in_q, :out_q

  def enq(data)
    in_q.enq(data)
  end

  def deq
    out_q.deq
  end

  def join!
    @thread.join
  end

private

  def spawn_reader(port)
    @thread = Thread.new(in_q, out_q) do |en_q, de_q|
      serv = TCPServer.new port
      loop do
        # Create a parser object
        parser = RequestParser.new

        # Pop a connection off the stack
        client = serv.accept

        # retrieve the headers
        lines = []
        while (line = client.gets) != "\r\n"
          # XXX Hack around missing CONTENT_LENGTH in parser.env
          if parser.env['CONTENT_LENGTH'].nil? && line =~ /^Content-Length: (\d+)/
            parser.env['CONTENT_LENGTH'] = $1.to_i
          elsif line =~ /(\S+): (.*)/
            parser.env[$1.upcase] = $2.chomp
          end
          lines << line
        end


        parser.parse(lines.join("\r\n"))

        if parser.content_length > 0
          parser.parse(client.gets(parser.content_length))
        end

        # Send the request object back to the caller
        de_q.enq(parser)

        # Fetch the response object from the caller
        response = en_q.deq
        client.write(response.to_s)

        client.close
      end
    end
  end

end

{'fraction_non_alphanumeric': 0.11254295532646048, 'fraction_numerical': 0.004009163802978236, 'mean_word_length': 2.1271262309758283, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7522773', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1135, 'n_words': 301}