<commit_before>require 'geoplanet'
require 'yaml'
require 'pry'

# Constant
APPID = ENV['GEOPLANET_APPID']
PLACE_ATTR = {
  lang: 'ja',
  count: 0,
  type: [8,12],
  select: 'long'
}
JAPAN_WOEID = 23424856

# Argument
place_woeid = (ARGV[0] || JAPAN_WOEID).to_i
file_name = (ARGV[1] || "geoplanet.yml")
debug_mode = (ARGV[2] || false)

class MyPlace < GeoPlanet::Place
  attr_accessor :parent, :chidlren

  def to_h(parent)
    {
      self.woeid => Hash[instance_variables.map {|i| [i.to_s.sub("@", ""), instance_variable_get(i)] }]
    }
    # hash[self.woeid].merge! Hash[instance_variables.map {|i| [i.to_s.sub("@", ""), instance_variable_get(i)] }]
    # return hash
  end

  def children(*args)
    p "do children"
    # @children ||= super(*args).map {|c| c.parent = self.woeid}
    children = super(*args)
    @children = children.map {|c| [c.woeid, c] }.to_h
    return children
  end
end

def get_children_tree(place, parent, _array, _tree)
  yield place if block_given?

  place.children
  place.parent = parent ? parent.woeid : nil
  hash = place.to_h(parent)

  binding.pry

  _array.merge!(hash)
  _tree.merge!(hash)

  children = place.children(PLACE_ATTR) || []
  children.each do |c|
    get_children_tree(c, place, _array, _tree[place.woeid]["children"]) {|place| yield place if block_given?}
  end
end

# main
GeoPlanet.appid = APPID
GeoPlanet.debug = debug_mode

place = MyPlace.new(place_woeid, [PLACE_ATTR.assoc(:lang), PLACE_ATTR.assoc(:select)].to_h )
binding.pry

array_hash = {}
tree_hash = {}
$place_num = 0
get_children_tree(place, nil, array_hash, tree_hash) { |place|
  $place_num += 1
  puts "fetching (" + $place_num.to_s + "): " + place.name + ", " + place.placetype
}

file_path = Dir.pwd + "/" + file_name
File.write(file_path, array_hash.to_yaml)

puts file_path + " successfully generated."<commit_msg>move monkey patch to gem.<commit_after>require 'geoplanet'
require 'yaml'
require 'pry'

# Constant
APPID = ENV['GEOPLANET_APPID']
PLACE_ATTR = {
  lang: 'ja',
  count: 0,
  type: [8,12],
  select: 'long'
}
JAPAN_WOEID = 23424856

# Argument
place_woeid = (ARGV[0] || JAPAN_WOEID).to_i
file_name = (ARGV[1] || "geoplanet.yml")
debug_mode = (ARGV[2] || false)

class MyPlace < GeoPlanet::Place
  attr_accessor :parent, :chidlren

  def to_h
    { self.woeid => Hash[instance_variables.map {|i| [i.to_s.sub("@", ""), instance_variable_get(i)] }] }
  end

  def children(*args)
    p "do children"
    # @children ||= super(*args).map {|c| c.parent = self.woeid}
    children = super(*args)
    @children = children.map {|c| [c.woeid, c] }.to_h
    return children
  end
end

def get_children_tree(place, parent)
  place.children
  place.parent = parent ? parent.woeid : nil

  yield place if block_given?

  children = place.children(PLACE_ATTR) || []
  children.each do |c|
    get_children_tree(c, place) {|place| yield place if block_given?}
  end
end

# main
GeoPlanet.appid = APPID
GeoPlanet.debug = debug_mode

place = MyPlace.new(place_woeid, [PLACE_ATTR.assoc(:lang), PLACE_ATTR.assoc(:select)].to_h )
binding.pry

array_hash = {}
$place_num = 1
get_children_tree(place, nil) { |place|
  puts "fetching (" + $place_num.to_s + "): " + place.name + ", " + place.placetype
  array_hash.merge!(place.to_h)
  $place_num += 1
}

file_path = Dir.pwd + "/" + file_name
File.write(file_path, array_hash.to_yaml)

puts file_path + " successfully generated."
{'fraction_non_alphanumeric': 0.15566037735849056, 'fraction_numerical': 0.010023584905660377, 'mean_word_length': 4.228043143297381, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '322694', 'n_tokens_mistral': 1398, 'n_tokens_neox': 1352, 'n_words': 271}