<commit_before>require 'httparty'
require_relative 'landmark/meituan'
require_relative 'landmark/baidu_waimai'
require_relative 'landmark/dianping'
require_relative 'landmark/elong_flight'
require_relative 'landmark/train'
require_relative 'landmark/metro'
require_relative 'landmark/scene'
require_relative 'landmark/university'
module Db
  class BasePoiLandmark < ActiveRecord::Base
  end
  class BaseElongHotelCity < ActiveRecord::Base
  end
end
module POI
  module LandMark
    class Worker
      Keys = %w(<KEY>)

      def initialize(web_site)
        @cp           = {
          :meituan      => MeiTuan,
          :baidu_waimai => BaiduWaimai,
          :dianping     => DianPing,
          :elong_flight => ElongFlight,
          :train        => Train,
          :metro        => Metro,
          :scene        => Scene,
          :university   => University,
        } 
        @index        =  0
        @pipe         =  Queue.new
        @geocoder     =  Baidumap::Request::Geocoder.new(Keys[@index])
        @crawler      =  @cp[web_site.to_sym].new
        @landmarks    =  Db::BasePoiLandmark
        @elong_cities =  Db::BaseElongHotelCity
      end 

      def producer
        Thread.new {
          city_list = @crawler.city_list
          city_list.each do |city|
            elong_city = @elong_cities.find_by(NameShort: city[:city_cn])
            pois       = @crawler.landmarks(city)
            pois.each do |name, city_info|
              puts "Fetching pois: #{name}'s location, type: #{city_info[:cata]}, city: #{city_info[:city_cn]}"
              begin
                addrs = location(name, city_info)
                redo if addrs.nil?
                @pipe << {
                  :name          => name,
                  :elong_city_id => elong_city.nil? ? '0000' : elong_city[:Code],
                }.merge(city_info).merge(addrs)
                puts "\e[32mFinished!\e[0m"
              rescue=>e
                warn e
                warn e.backtrace
                next
              end
            end
          end
        }
      end

      def consumer 
        Thread.new {
          begin
          while @pduer.status or @pipe.length>0 do 
            row     =  @pipe.pop
            existed = @landmarks.find_by(name: row[:name],city_cn: row[:city_cn],source_domain: row[:source_domain])
            existed.nil? ? @landmarks.new(row).save : existed.update(row)
            sleep(1/(@pipe.length+1))
          end
          rescue=>e
            warn e
            warn e.backtrace
            exit
          end
        }
      end

      def work
        @pduer  = producer
        @writer = consumer
        @pduer.join
        @writer.join
        abort( "Congratulation! All things Finished.")
      end 

      def check_city(geo_dtls, city)
        addr  = geo_dtls['addressComponent']
        [addr['city'], addr['district']].any? {|ad| ad.include?(city) }
      end

      def location(name, city)
        addr_dtls = {:lng=>nil,:lat=>nil,:address=>nil,:province=>nil}
        geo_crd   = @geocoder.encode(name, city[:city_cn]).result

        if geo_crd.nil?
          @geocoder = change_key  # change_key if quotas runs out
          return nil  
        elsif !geo_crd.empty?
          lng_lat   = geo_crd['location']
          geo_dtls  = @geocoder.decode(lng_lat['lat'],lng_lat['lng']).result

          if check_city(geo_dtls,city[:city_cn])
            addr_dtls = {
              :lng      => lng_lat['lng'],
              :lat      => lng_lat['lat'],
              :address  => geo_dtls['formatted_address'],
              :province => geo_dtls['addressComponent']['province'],
            }
          end
        end
        addr_dtls
      end
       
      # Change key
      def change_key
        @index+=1
        raise "No more keys remains." if @index>=Keys.length
        @geocoder = Baidumap::Request::Geocoder.new(Keys[@index])
      end

    end
  end<commit_msg>fetch city_cn using baidumap api<commit_after>require 'httparty'
require_relative 'landmark/meituan'
require_relative 'landmark/baidu_waimai'
require_relative 'landmark/dianping'
require_relative 'landmark/elong_flight'
require_relative 'landmark/train'
require_relative 'landmark/metro'
require_relative 'landmark/scene'
require_relative 'landmark/university'
module Db
  class BasePoiLandmark < ActiveRecord::Base
  end
  class BaseElongHotelCity < ActiveRecord::Base
  end
end
module POI
  module LandMark
    class Worker
      Keys = %w(<KEY>)

      def initialize(web_site)
        @cp           = {
          :meituan      => MeiTuan,
          :baidu_waimai => BaiduWaimai,
          :dianping     => DianPing,
          :elong_flight => ElongFlight,
          :train        => Train,
          :metro        => Metro,
          :scene        => Scene,
          :university   => University,
        } 
        @index        =  0
        @pipe         =  Queue.new
        @geocoder     =  Baidumap::Request::Geocoder.new(Keys[@index])
        @crawler      =  @cp[web_site.to_sym].new
        @landmarks    =  Db::BasePoiLandmark
        @elong_cities =  Db::BaseElongHotelCity
      end 

      def producer
        Thread.new {
          city_list   = @crawler.city_list 
          city_list.each do |city|
            city_cn     = city[:city_cn]
            @elong_city = @elong_cities.find_by(NameShort: city_cn)
            pois        = @crawler.landmarks(city)
            pois.each do |name, city_info|
              puts "Fetching pois: #{name}'s location, type: #{city_info[:cata]}, city: #{city_info[:city_cn]}"
              begin
                addrs = location(name, city_cn)
                redo if addrs.nil?
                @pipe << {
                  :name          => name,
                  :elong_city_id => @elong_city.nil? ? '0000' : @elong_city[:Code],
                }.merge(city_info).merge(addrs)
                puts "\e[32mFinished!\e[0m"
              rescue=>e
                warn "#{e}\n#{e.backtrace.join("\n")}"
                next
              end
            end
          end
        }
      end

      def consumer 
        Thread.new {
          begin
          while @pduer.status or @pipe.length>0 do 
            row     =  @pipe.pop
            existed = @landmarks.find_by(name: row[:name],city_cn: row[:city_cn],source_domain: row[:source_domain])
            existed.nil? ? @landmarks.new(row).save : existed.update(row)
            sleep(1/(@pipe.length+1))
          end
          rescue=>e
            warn "#{e}\n#{e.backtrace.join("\n")}"
            exit
          end
        }
      end

      def work
        @pduer  = producer
        @writer = consumer
        @pduer.join
        @writer.join
        abort( "Congratulation! All things Finished.")
      end 

      def check_city(geo_dtls, city)
        addr  = geo_dtls['addressComponent']
        [addr['city'], addr['district']].any? { |ad| city.nil? or ad.include?(city.to_s) }
      end

      def location(name, city)
        addr_dtls = {:lng=>nil,:lat=>nil,:address=>nil,:province=>nil}
        geo_crd   = @geocoder.encode(name, city).result

        if geo_crd.nil?
          @geocoder = change_key  # change_key if quotas runs out
          return
        elsif !geo_crd.empty?
          lng_lat   = geo_crd['location']
          geo_dtls  = @geocoder.decode(lng_lat['lat'],lng_lat['lng']).result

          if check_city(geo_dtls, city)
            addr_dtls = {
              :lng      => lng_lat['lng'],
              :lat      => lng_lat['lat'],
              :city_cn  => geo_dtls['addressComponent']['city'].gsub(/市$/,''),
              :address  => geo_dtls['formatted_address'],
              :province => geo_dtls['addressComponent']['province'],
            }
          end
        end
        @elong_city = @elong_cities.find_by(NameShort: addr_dtls[:city_cn]) if addr_dtls[:city_cn]
        addr_dtls
      end
       
      # Change key
      def change_key
        @index+=1
        raise "No more keys remains." if @index>=Keys.length
        @geocoder = Baidumap::Request::Geocoder.new(Keys[@index])
      end

    end
  end
{'fraction_non_alphanumeric': 0.11336779911373708, 'fraction_numerical': 0.0029542097488921715, 'mean_word_length': 1.7274253105068815, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7616996', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2551, 'n_words': 474}