<commit_before>module CouchRest
  class Streamer
    attr_accessor :db
    def initialize db
      @db = db
    end
    
    # Stream a view, yielding one row at a time. Shells out to <tt>curl</tt> to keep RAM usage low when you have millions of rows.
    def view name, params = nil, &block
      urlst = if /^_/.match(name) then
        "#{@db.root}/#{name}"
      else
        name = name.split('/')
        dname = name.shift
        vname = name.join('/')
        "#{@db.root}/_design/#{dname}/_view/#{vname}"
      end
      url = CouchRest.paramify_url urlst, params
      # puts "stream #{url}"
      first = nil
      IO.popen("curl --silent \"#{url}\"") do |view|
        first = view.gets # discard header
        while line = view.gets 
          row = parse_line(line)<commit_msg>use single quotes when passing the URL to curl<commit_after>module CouchRest
  class Streamer
    attr_accessor :db
    def initialize db
      @db = db
    end
    
    # Stream a view, yielding one row at a time. Shells out to <tt>curl</tt> to keep RAM usage low when you have millions of rows.
    def view name, params = nil, &block
      urlst = if /^_/.match(name) then
        "#{@db.root}/#{name}"
      else
        name = name.split('/')
        dname = name.shift
        vname = name.join('/')
        "#{@db.root}/_design/#{dname}/_view/#{vname}"
      end
      url = CouchRest.paramify_url urlst, params
      # puts "stream #{url}"
      first = nil
      IO.popen("curl --silent '#{url}'") do |view|
        first = view.gets # discard header
        while line = view.gets 
          row = parse_line(line)
{'fraction_non_alphanumeric': 0.12739641311069883, 'fraction_numerical': 0.0, 'mean_word_length': 2.0702087286527515, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1057357', 'n_tokens_mistral': 522, 'n_tokens_neox': 517, 'n_words': 165}