<commit_before>require 'open-uri'
require 'nokogiri'

require './nodeparser.rb'

url = 'https://news.ycombinator.com/'
charset = nil
html = open(url) do |f|
  charset = f.charset
  f.read
end

doc = Nokogiri::HTML.parse(html, nil, charset)
np = NodeParser.new('#\31 3936153 > td:nth-child(3) > a text{}')
p np.parse(doc)<commit_msg>update test code<commit_after>require 'open-uri'
require 'nokogiri'
require 'json'

require './nodeparser.rb'

file = File.read('sample/hacker_news.json')
struct = JSON.parser.new(file).parse
# url = 'https://news.ycombinator.com/'
# charset = nil
# html = open(url) do |f|
#   charset = f.charset
#   f.read
# end
# doc = Nokogiri::HTML.parse(html, nil, charset)

def serializer(struct)
  out = Hash.new
  struct.each do |key, value|
    p '-' * 30
    tkey = ''
    if match = key.match(/(?<k1>([^:]*)):`(?<k2>([^`]*))`/)
      if is_array = match[:k1].match(/\[\].*/)
        p 'array key'
      end
      tkey = match[:k1]
      p match[:k2]
    else
      raise InvalidSyntaxKey
    end

    case value.to_s
    when 'string'
      p 'string'
    when /^{.*}$/
      p 'object'
      serializer(struct[key])
    else
      raise InvalidSyntaxValue
    end
  end

  out
end

serializer(struct)
{'fraction_non_alphanumeric': 0.17331163547599673, 'fraction_numerical': 0.013832384052074858, 'mean_word_length': 2.917197452229299, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '149226', 'n_tokens_mistral': 488, 'n_tokens_neox': 464, 'n_words': 106}